"""
Compile **necsim** with default or provided compilation options. Intended for internal usage during ``pip`` or ``conda``
builds, although manual installation is also possible by running this file from the command line.
``python installer.py`` configures the install by detecting system components and compiles the ``C++`` files, if
possible. Command line flags can be provided to installer.py to modify the install (see
:ref:`Compilation Options <sec Compilation Options>` for more information).
"""
from __future__ import print_function, absolute_import, division # Only Python 2.x
import itertools
import logging
import os
import platform
import re
import shutil
import subprocess
import sys
import time
from distutils import sysconfig
# Import system operations for subprocess executation and log file handling
try:
from .future_except import FileNotFoundError
from .system_operations import execute_log_info, set_logging_method, execute_silent, mod_directory
except (ImportError, SystemError, ValueError): # pragma: no cover
from future_except import FileNotFoundError
from system_operations import execute_log_info, set_logging_method, execute_silent, mod_directory
from setuptools.command.build_ext import build_ext
[docs]class Installer(build_ext): # pragma: no cover
"""Wraps configuration and compilation of C++ code."""
def __init__(self, dist, **kwargs):
"""Generates the link to the mod directory for installation."""
build_ext.__init__(self, dist)
self.mod_dir = mod_directory
self.build_dir = None
self.obj_dir = None
[docs] def make_depend(self):
"""
Runs make depend in the lib directory to calculate all dependencies for the header and source files.
.. note:: Fails silently if makedepend is not installed, printing an error to logging.
"""
try:
execute_silent(["make", "depend"], cwd=os.path.join(self.mod_dir, "lib/"))
except (RuntimeError, subprocess.CalledProcessError) as rte:
logging.error("Could not execute makedepend: " + str(rte))
logging.error("Using default dependencies instead.")
self.use_default_depends()
[docs] def create_default_depend(self):
"""
Runs the default makedepend command, outputting dependencies to lib/depends_default.
Used to generate a default dependency file on a system where makedepend exists, for a system where it does not.
"""
try:
execute_silent(
["makedepend", "*.cpp", "necsim/*.cpp", "-f" "depends_default", "-p", "obj/", "-Y"],
cwd=os.path.join(self.mod_dir, "lib/"),
)
except (RuntimeError, subprocess.CalledProcessError) as rte:
logging.error("Could not execute makedepend: " + str(rte))
[docs] def use_default_depends(self):
"""
Uses the default dependencies, copying all contents of depends_default to the end of Makefile.
.. note:: Zero error-checking is done here as the Makefiles should not change, and the depends_default file should
be created using create_default_depend()
"""
# First read Makefile
with open(os.path.join(self.mod_dir, "lib", "Makefile"), "r") as f_in:
lines = f_in.readlines()
with open(os.path.join(self.mod_dir, "lib", "depends_default"), "r") as f_default:
lines_default = f_default.readlines()
# Now write all back out
with open(os.path.join(self.mod_dir, "lib/Makefile"), "w") as f_out:
for line in lines:
if "# DO NOT DELETE" in line:
break
f_out.write(line)
for line in lines_default:
f_out.write(line)
[docs] def do_compile(self):
"""
Compiles the C++ necsim program by running make. This changes the working directory to wherever the module has been
installed for the subprocess call.
"""
# Check that the make file exists
if os.path.exists(os.path.join(self.mod_dir, "lib/")):
time.sleep(0.5)
self.make_depend()
# Sleep to ensure that file timings are updated (support for HPC systems with inaccurate timings).
time.sleep(1)
try:
execute_log_info(["make", "all"], cwd=os.path.join(self.mod_dir, "lib/"))
logging.info("Compilation exited successfully.")
except (RuntimeError, subprocess.CalledProcessError) as rte:
logging.error(str(rte))
logging.error("Compilation attempted, but error thrown.")
else:
raise IOError("C++ library does not exist! Check relative file path")
[docs] def move_shared_object_file(self):
"""
Moves the shared object (.so) file to the build directory.
:return:
"""
directory = os.path.join(self.mod_dir, "necsim")
dirv = "sharedpy" + sys.version[0]
for file in ["libnecsim.so"]:
src = os.path.join(self.mod_dir, "lib", file)
version_dir = os.path.join(directory, dirv)
if not os.path.exists(src):
raise IOError("Shared object file {} does not exists. Check installation was successful.".format(src))
if not os.path.exists(directory):
os.mkdir(directory)
if not os.path.exists(version_dir):
os.mkdir(version_dir)
if not os.path.exists(os.path.join(version_dir, "__init__.py")):
open(os.path.join(version_dir, "__init__.py"), "a").close()
dst = os.path.join(directory, dirv, file)
if os.path.exists(dst):
os.remove(dst)
shutil.copy(src, dst)
[docs] def get_build_dir(self):
"""
Gets the build directory.
:return: the build directory path
"""
if self.build_dir is None:
directory = os.path.join(self.mod_dir, "necsim")
return directory
return os.path.join(self.build_dir)
[docs] def get_obj_dir(self):
"""
Gets the obj directory for installing obj files to.
:return: the obj directory path
"""
if self.obj_dir is None:
return "obj"
return self.obj_dir
[docs] def autoconf(self):
"""
Runs the `autoconf` bash function (assuming that autoconf is available) to create the `configure` executable.
"""
try:
execute_log_info(["autoconf"], cwd=os.path.join(self.mod_dir, "lib/"))
except (RuntimeError, subprocess.CalledProcessError, FileNotFoundError, OSError, IOError) as cpe:
logging.warning(
"Could not run autoconf function to generate configure executable. "
"Please run this functionality manually if installation fails."
)
logging.warning(str(cpe))
[docs] def clean(self):
"""
Runs make clean in the NECSim directory to wipe any previous potential compile attempts.
"""
try:
time.sleep(0.5)
execute_log_info(["make", "obj_dir"], cwd=os.path.join(self.mod_dir, "lib/"))
execute_log_info(["make", "build_dir"], cwd=os.path.join(self.mod_dir, "lib/"))
execute_log_info(["make", "clean"], cwd=os.path.join(self.mod_dir, "lib/"))
except subprocess.CalledProcessError as cpe:
raise RuntimeError("Make file has not been generated. Cannot clean: ".format(cpe))
[docs] def get_ldflags(self):
"""Get the ldflags that Python was compiled with, removing some problematic options."""
sysldflags = sysconfig.get_config_var("LDFLAGS")
if sysldflags is None:
ldflags = ""
else:
ldflags = re.sub(r"-arch \b[^ ]*[\ ]*", "", sysldflags) + " "
if "--sysroot=" in ldflags:
logging.warning("--sysroot found in LDFLAGS, removing")
ldflags = re.sub(r"--sysroot=.*[,\s]", "", ldflags)
ldflags = ldflags.replace("\n", " ")
return ldflags
[docs] def get_ldshared(self):
"""Get the ldshared Python variables and replaces -bundle with -shared for proper compilation."""
ldflags = sysconfig.get_config_var("LDSHARED")
if ldflags is None:
return ""
ldflags = " ".join(ldflags.split()[1:]).replace("-bundle", "-shared")
return ldflags
[docs] def get_compilation_flags(self, display_warnings=False):
"""
Generates the compilation flags for passing to ./configure.
:param display_warnings: If true, runs with the -Wall flag for compilation (displaying all warnings). Default is False.
:return: list of compilation flags.
:rtype: list
"""
# Get the relevant flags that Python was originally compiled with, to be passed to the C++ code.
include = str("CPPFLAGS=-I" + sysconfig.get_python_inc()).replace("\n", "")
cflags = " " + sysconfig.get_config_var("CFLAGS")
cflags = str(re.sub(r"-arch \b[^ ]*", "", cflags)).replace("\n", "") # remove any architecture flags
cflags += " "
py_ldflags = str(
"-L"
+ sysconfig.get_python_lib(standard_lib=True)
+ " -L"
+ sysconfig.get_config_var("DESTDIRS").replace(" ", " -L")
).replace("\n", "")
py_lib = "PYTHON_LIB=-lpython"
ldflags = str("LDFLAGS=" + self.get_ldflags())
# Get the shared object platform-specific compilation flags.
platform_so = "PLATFORM_SO="
if platform.system() == "Linux":
platform_so += "-shared"
elif platform.system() == "Darwin":
platform_so += "-dynamiclib"
elif platform.system() == "Windows":
raise SystemError(
"COMPILATION FAILURE: Windows is not yet supported. You must compile the libraries yourself."
)
else:
logging.critical("OS not detected, compilation failures likely. Please report this error.")
# Make sure that the linker directs to the correct Python library (such as -lpython3.5m)
# Eventually this will also detect if the install is for an anaconda distribution.
if "conda" not in sys.version and "Continuum" not in sys.version:
py_lib += sys.version[0:3]
if "m" in sysconfig.get_config_var("LIBRARY"):
py_lib += "m"
else:
cflags += "-I{}/include {}".format(os.environ["PREFIX"], "--enable-shared")
ldflags += "-L{}/lib".format(os.environ["PREFIX"])
py_lib += sys.version[0:3]
if "m" in sysconfig.get_config_var("LIBRARY"):
py_lib += "m"
cflags += " -DANACONDA"
ldflags += " " + py_ldflags
py_ldflags = "PYTHON_LDFLAGS=" + py_ldflags
call = [include + cflags, py_lib, ldflags, py_ldflags, platform_so]
# Remove the flags which would potentially cause unnecessary warnings to be thrown.
# This can be disabled by passing display_warnings=True
if not display_warnings:
call = [re.sub("-Wstrict-prototypes|-Wno-unused-result|-Wunused-variable|-Wall", "", x) for x in call]
return call
[docs] def backup_makefile(self):
"""
Copies the makefile to a saved folder so that even if the original is overwritten, the last successful
compilation can be recorded.
"""
src = os.path.join(self.mod_dir, "lib", "Makefile")
dst = os.path.join(self.mod_dir, "reference", "Makefile")
if not os.path.exists(src):
logging.error("Makefile does not exist at {}. Check successful compilation".format(src))
return
if os.path.exists(dst):
os.remove(dst)
shutil.copy2(src, dst)
[docs] def copy_makefile(self):
"""
Copies the backup makefile to the main directory, if it exists.
Throws an IOError if no makefile is found.
"""
src = os.path.join("reference", "Makefile")
dst = os.path.join("lib", "Makefile")
if not os.path.exists(src):
raise IOError("Cannot find backup Makefile, requires running of configuration scripts.")
if os.path.exists(dst):
os.remove(dst)
shutil.copy2(src, dst)
[docs] def run_cmake(self, src_dir, cmake_args, build_args, tmp_dir, env):
"""
Runs cmake to compile necsim.
:param src_dir: the source directory for necsim .cpp and .h files
:param cmake_args: arguments to pass to the cmake project
:param tmp_dir: the build directory to output cmake files to
:param env: the os.environ (or other environmental variables) to pass on
"""
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
try:
subprocess.check_call(["cmake", src_dir] + cmake_args, cwd=tmp_dir, env=env)
if platform.system() == "Windows":
subprocess.check_call(["nmake"], cwd=tmp_dir, env=env)
# subprocess.check_call(["nmake", "install"], cwd=tmp_dir, env=env)
else:
subprocess.check_call(
["cmake", "--build", ".", "--target", "necsim"] + build_args, cwd=tmp_dir, env=env
)
except subprocess.CalledProcessError as cpe:
raise SystemError("Fatal error running cmake in directory: {}".format(cpe))
if platform.system() == "Windows":
shutil.copy(os.path.join(tmp_dir, "necsim.pyd"), os.path.join(self.get_build_dir(), "libnecsim.pyd"))
[docs] def run(self):
"""Runs installation and generates the shared object files - entry point for setuptools"""
for ext in self.extensions:
self.build_extension(ext)
[docs] def build_extension(self, ext):
"""Builds the C++ and Python extension."""
self.build_dir = os.path.abspath(
os.path.join(os.path.dirname(self.get_ext_fullpath(ext.name)), "pycoalescence", "necsim")
)
self.obj_dir = self.build_dir
if not os.path.exists(self.build_temp):
os.makedirs(self.build_temp)
self.setuptools_cmake(ext)
[docs] def clean_cmake(self):
"""Deletes the cmake files and object locations if they exist."""
for path in [
os.path.join(self.get_build_dir(), "libnecsim.so"),
os.path.join(self.get_build_dir(), "libnecsim.so.dSYM"),
os.path.join(self.get_build_dir(), "libnecsim.pyd"),
os.path.join(self.get_build_dir(), "libnecsim.dylib"),
os.path.join(self.mod_dir, "obj"),
]:
if os.path.exists(path):
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
[docs] def get_default_cmake_args(self, output_dir):
"""
Returns the default cmake configure and build arguments.
:param output_dir: the output directory to use
:return: tuple of two lists, first containing cmake configure arguments, second containing build arguments
:rtype: tuple
"""
cfg = "Debug" if self.debug else "Release"
cflags = sysconfig.get_config_var("CFLAGS")
if cflags is not None:
cflags = str(re.sub(r"-arch \b[^ ]*", "", cflags)).replace("\n", "") # remove any architecture flags
else:
cflags = ""
gdal_inc_path = None
gdal_dir = None
if platform.system() == "Windows":
libdir = get_python_library("{}.{}".format(sys.version_info.major, sys.version_info.minor))
else:
conda_prefix = os.environ.get("PREFIX", None) # for conda only under unix
libdir = sysconfig.get_config_var("LIBDIR")
if conda_prefix is not None:
gdal_inc_path = os.path.join(conda_prefix, "include")
gdal_dir = os.path.join(conda_prefix, "lib")
else:
try:
gdal_inc_path = subprocess.check_output(["gdal-config", "--cflags"], env=os.environ)
gdal_dir = subprocess.check_output(["gdal-config", "--prefix"], env=os.environ)
except subprocess.CalledProcessError:
pass
if gdal_inc_path is not None:
gdal_inc_path = gdal_inc_path.decode("utf-8").split(" ")[0][2:].replace("\n", "")
if gdal_dir is not None:
gdal_dir = gdal_dir.decode("utf-8").replace("\n", "")
if libdir is None:
libdir = os.path.abspath(os.path.join(sysconfig.get_config_var("LIBDEST"), "..", "libs"))
if sysconfig.get_config_var("LIBDEST") is None:
raise SystemError("Cannot detect library directory for Python install.")
cmake_args = [
"-DPYTHON_LIBRARY:FILEPATH={}".format(libdir),
"-DPYTHON_CPPFLAGS:='{}'".format(cflags),
"-DPYTHON_LDFLAGS:='{}'".format(self.get_ldshared()),
"-DPYTHON_INCLUDE_DIR:FILEPATH={}".format(sysconfig.get_python_inc()),
"-DCMAKE_BUILD_TYPE={}".format(cfg),
]
if gdal_inc_path is not None:
cmake_args.append("-DGDAL_INCLUDE_DIR={}".format(gdal_inc_path))
if gdal_dir is not None:
cmake_args.append("-DGDAL_DIR={}".format(gdal_dir))
build_args = ["--config", cfg]
if platform.system() == "Windows":
cmake_args += [
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY:PATH={}".format(output_dir),
"-DCMAKE_PREFIX_PATH:PATH={}".format(os.environ.get("LIBRARY_PREFIX", "")),
"-DCMAKE_INSTALL_PREFIX:PATH={}".format(os.environ.get("LIBRARY_PREFIX", "")),
"-DCMAKE_BUILD_TYPE:STRING={}".format(cfg),
"-G",
"NMake Makefiles",
]
# if sys.maxsize > 2 ** 32: # Removed to allow for building on conda-forge
# cmake_args += ["-A", "x64"]
build_args += ["--", "/m"]
else:
cmake_args += [
"-DCMAKE_BUILD_TYPE={}".format(cfg),
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY:PATH={}".format(output_dir),
]
build_args += ["--", "-j2"]
return cmake_args, build_args
[docs]def get_python_library(python_version): # pragma: no cover
"""Get path to the Python library associated with the current Python
interpreter."""
# determine direct path to libpython
python_library = sysconfig.get_config_var("LIBRARY")
potential_library = None
# if static (or nonexistent), try to find a suitable dynamic libpython
if python_library is None or python_library[-2:] == ".a":
candidate_lib_prefixes = ["", "lib"]
candidate_extensions = [".lib", ".so"]
if sysconfig.get_config_var("WITH_DYLD"):
candidate_extensions.insert(0, ".dylib")
candidate_versions = [python_version]
if python_version:
candidate_versions.append("")
candidate_versions.insert(0, "".join(python_version.split(".")[:2]))
abiflags = getattr(sys, "abiflags", "")
candidate_abiflags = [abiflags]
if abiflags:
candidate_abiflags.append("")
# Ensure the value injected by virtualenv is
# returned on windows.
# Because calling `sysconfig.get_config_var('multiarchsubdir')`
# returns an empty string on Linux, `du_sysconfig` is only used to
# get the value of `LIBDIR`.
libdir = sysconfig.get_config_var("LIBDIR")
if sysconfig.get_config_var("MULTIARCH"):
masd = sysconfig.get_config_var("multiarchsubdir")
if masd:
if masd.startswith(os.sep):
masd = masd[len(os.sep):]
libdir = os.path.join(libdir, masd)
if libdir is None:
libdir = os.path.abspath(os.path.join(sysconfig.get_config_var("LIBDEST"), "..", "libs"))
candidates = (
os.path.join(libdir, "".join((pre, "python", ver, abi, ext)))
for (pre, ext, ver, abi) in itertools.product(
candidate_lib_prefixes, candidate_extensions, candidate_versions, candidate_abiflags
)
)
for candidate in candidates:
if os.path.exists(candidate):
# we found a (likely alternate) libpython
potential_library = candidate
if potential_library[-2:] != ".a":
break
# Otherwise still a static library, keep searching
if potential_library is None:
raise IOError("No Python libraries found")
return potential_library
if __name__ == "__main__": # pragma: no cover
fail = True
from distutils.dist import Distribution
import argparse
parser = argparse.ArgumentParser(description="Build the C++ library (necsim) required for pycoalescence.")
parser.add_argument("--cmake", action="store_true", default=True, dest="cmake", help="use the cmake build process")
parser.add_argument(
"--autotools",
action="store_true",
default=False,
dest="autotools",
help="Use the autotools build process (./configure and make)",
)
parser.add_argument(
"--compiler-args",
metavar="N",
type=str,
nargs="+",
dest="compiler_args",
default=[],
help="Additional arguments to pass to the autotools compiler",
)
parser.add_argument(
"--cmake-args",
metavar="N",
type=str,
nargs="+",
dest="cmake_args",
default=[],
help="Additional arguments to pass to the cmake compiler during configuration",
)
parser.add_argument(
"--cmake-build-args",
metavar="N",
type=str,
nargs="+",
dest="cmake_build_args",
default=[],
help="Additional arguments to pass to the cmake compiler at build time",
)
parser.add_argument("--debug", action="store_true", default=False, dest="debug", help="Compile using DEBUG defines")
parser.add_argument(
"-c",
"-C",
"--compile",
action="store_true",
default=False,
dest="compile_only",
help="Compile only, do not re-configure necsim",
)
parser.add_argument(
"--clean",
action="store_true",
default=False,
dest="clean",
help="Clean previous cmake builds from this directory.",
)
args = parser.parse_args()
if args.cmake and args.autotools:
raise ValueError("Cannot use both cmake and autotools build process - specify one or the other.")
if not args.cmake or args.autotools:
raise ValueError("Must specify compilation either using autotools or cmake.")
dist = Distribution()
installer = Installer(dist)
installer.debug = args.debug
if args.clean:
installer.clean_cmake()
else:
if args.cmake:
env = os.environ.copy()
build_dir = installer.get_build_dir()
obj_dir = installer.get_obj_dir()
cmake_args, build_args = installer.get_default_cmake_args(build_dir)
cmake_args += args.cmake_args
build_args += args.cmake_build_args
src_dir = os.path.join(installer.mod_dir, "lib")
installer.run_cmake(src_dir, cmake_args, build_args, obj_dir, env)
else:
if platform.system() != "Windows":
raise SystemError("Usage of configure and make on a windows system is not supported.")
if args.compile_only:
set_logging_method(logging_level=logging.INFO)
installer.copy_makefile()
installer.do_compile()
fail = False
if fail:
installer.configure_and_compile(args.compiler_args)