#! /usr/bin/env python3 import logging import platform import subprocess import sys import os import shutil from argparse import ArgumentParser, RawDescriptionHelpFormatter import argparse from collections import OrderedDict import atexit import json import pprint import shlex from glob import iglob from itertools import chain import re import importlib try: from pkg_resources.extern.packaging.version import Version, InvalidVersion except ModuleNotFoundError: from packaging.version import Version, InvalidVersion osname = platform.uname().system arch = platform.uname().machine # Packages which we wish to ensure are always recompiled wheel_blacklist = ["mpi4py", "randomgen", "numpy"] # Packages for which we set CC=mpicc, CXX=mpicxx, F90=mpif90. parallel_packages = ["h5py", "petsc4py", "slepc", "slepc4py", "PyOP2", "libsupermesh", "firedrake"] # Firedrake application installation shortcuts. firedrake_apps = { "gusto": ("""Atmospheric dynamical core library. http://firedrakeproject.org/gusto""", "git+ssh://github.com/firedrakeproject/gusto.git@main#egg=gusto"), "thetis": ("""Coastal ocean model. http://thetisproject.org""", "git+ssh://github.com/thetisproject/thetis#egg=thetis"), "icepack": ("""Glacier and ice sheet model. https://icepack.github.io""", "git+ssh://github.com/icepack/icepack.git#egg=icepack"), "irksome": ("""Implicit Runge-Kutta methods. https://github.com/firedrakeproject/Irksome/""", "git+ssh://github.com/firedrakeproject/Irksome.git#egg=Irksome"), "femlium": ("""Interactive visualization of finite element simulations on geographic maps with folium. https://femlium.github.io/""", "git+ssh://github.com/FEMlium/FEMlium.git@main#egg=FEMlium"), "fascd": ("""Full approximation scheme constraint decomposition solver for variational inequalities. https://bitbucket.org/pefarrell/fascd""", "git+ssh://bitbucket.org/pefarrell/fascd.git@master#egg=fascd"), "defcon": ("""Deflated continuation algorithm for bifurcation analysis. https://bitbucket.org/pefarrell/defcon""", "git+ssh://bitbucket.org/pefarrell/defcon.git@master#egg=defcon"), "gadopt": ("""Geodynamic Adjoint Optimization Platform. http://g-adopt.github.io""", "git+ssh://github.com/g-adopt/g-adopt.git@master#egg=gadopt"), "asQ": ("""ParaDiag parallel-in-time methods. http://firedrakeproject.org/asQ""", "git+ssh://github.com/firedrakeproject/asQ.git@master#egg=asQ"), } class InstallError(Exception): # Exception for generic install problems. pass class FiredrakeConfiguration(dict): """A dictionary extended to facilitate the storage of Firedrake configuration information.""" def __init__(self, args=None): super(FiredrakeConfiguration, self).__init__() '''A record of the persistent options in force.''' self["options"] = {} '''Relevant environment variables.''' self["environment"] = {} '''Additional packages installed via the plugin interface.''' self["additions"] = [] if args: for o in self._persistent_options: if o in args.__dict__.keys(): self["options"][o] = args.__dict__[o] _persistent_options = ["package_manager", "minimal_petsc", "mpicc", "mpicxx", "mpif90", "mpiexec", "disable_ssh", "honour_petsc_dir", "with_parmetis", "slepc", "packages", "honour_pythonpath", "opencascade", "tinyasm", "torch", "petsc_int_type", "cache_dir", "complex", "remove_build_files", "with_blas", "netgen"] def deep_update(this, that): from collections import abc for k, v in that.items(): if isinstance(v, abc.Mapping) and k in this.keys(): this[k] = deep_update(this.get(k, {}), v) else: this[k] = v return this if os.path.basename(__file__) == "firedrake-install": mode = "install" logfile_directory = os.path.abspath(os.getcwd()) logfile_mode = "w" elif os.path.basename(__file__) == "firedrake-update": mode = "update" os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../..") try: logfile_directory = os.environ["VIRTUAL_ENV"] except KeyError: quit("Unable to retrieve venv name from the environment.\n Please ensure the venv is active before running firedrake-update.") logfile_mode = "a" if "--no-update-script" in sys.argv else "w" else: sys.exit("Script must be invoked either as firedrake-install or firedrake-update") # Set up logging # Log to file at DEBUG level if ("-h" in sys.argv) or ("--help" in sys.argv): # Don't log if help displayed to avoid overwriting an existing log logfile = os.devnull else: logfile = os.path.join(logfile_directory, 'firedrake-%s.log' % mode) logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-6s %(message)s', filename=logfile, filemode=logfile_mode) # Log to console at INFO level console = logging.StreamHandler() console.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') console.setFormatter(formatter) logging.getLogger().addHandler(console) log = logging.getLogger() log.info("Running %s" % " ".join(sys.argv)) if sys.version_info >= (3, 12): print("""\nCan not install Firedrake with Python 3.12 at the moment: Some wheels are not yet available for Python 3.12 for some required package(s). Please install with Python 3.11 (or an earlier version >= 3.8).""") sys.exit(1) elif sys.version_info < (3, 9) and osname == "Darwin" and arch == "arm64": print(""" Installing Firedrake on Mac Arm (M1 or M2) requires at least Python 3.9 since some required package(s) are not available for earlier Python versions.""") elif sys.version_info < (3, 8): if mode == "install": print("""\nInstalling Firedrake requires Python 3, at least version 3.8. You should run firedrake-install with python3.""") if mode == "update": if hasattr(sys, "real_prefix"): # sys.real_prefix exists iff we are in an active virtualenv. # # Existing install trying to update past the py2/py3 barrier print("""\nFiredrake is now Python 3 only. You cannot upgrade your existing installation. Please follow the instructions at http://www.firedrakeproject.org/download.html to reinstall.""") sys.exit(1) else: # Accidentally (?) running firedrake-update with python2. print("""\nfiredrake-update must be run with Python 3, did you accidentally use Python 2?""") sys.exit(1) branches = {} def resolve_doi_branches(doi): import requests import hashlib log.info("Installing Firedrake components specified by {}".format(doi)) # Zenodo updated their API to use elastic search so simply querying with # ?q=doi:10.5281/zenodo. now returns multiple results (not ideal) # Instead use # / # as documented here https://developers.zenodo.org/#retrieve id_ = doi.split('.')[-1] response = requests.get("https://zenodo.org/api/records/{}".format(id_)) if not response.ok: log.error("Unable to obtain Zenodo record for doi {}".format(doi)) log.error("Response was {}".format(response.json())) sys.exit(1) record = response.json() files = record["files"] try: componentjson, = (f for f in files if f["key"] == "components.json") except ValueError: log.error("Expecting to find exactly one 'components.json' in record") sys.exit(1) download = requests.get(componentjson["links"]["self"]) if download.status_code >= 400: log.error("Unable to download 'components.json'") log.error("Response was {}".format(download.json())) sys.exit(1) # component response has checksum as "md5:HEXDIGEST", strip the md5. if hashlib.md5(download.content).hexdigest() != componentjson["checksum"][4:]: log.error("Download failed checksum, expecting {expect}, got {got}".format( expect=componentjson["checksum"][4:], got=hashlib.md5(download.content).hexdigest())) sys.exit(1) componentjson = download.json() branches = {} for record in componentjson["components"]: commit = record["commit"] component = record["component"] package = component[component.find("/")+1:].lower() branches[package] = commit log.info("Using commit {commit} for component {comp}".format( commit=commit, comp=component)) return branches def honour_petsc_dir_get_petsc_dir(): try: petsc_dir = os.environ["PETSC_DIR"] except KeyError: raise InstallError("Unable to find installed PETSc (did you forget to set PETSC_DIR?)") petsc_arch = os.environ.get("PETSC_ARCH", "") return petsc_dir, petsc_arch def honour_petsc_dir_fetch_petscconf(name): # Return the line in "petscconf.h" that starts with name. petsc_dir, petsc_arch = honour_petsc_dir_get_petsc_dir() petscconf_h = os.path.join(petsc_dir, petsc_arch, "include", "petscconf.h") with open(petscconf_h) as fh: for line in fh.readlines(): if line.startswith(name): return line def honour_petsc_dir_get_petsc_int_type(): line = honour_petsc_dir_fetch_petscconf("#define PETSC_USE_64BIT_INDICES") if line and line.split()[2] == '1': return "int64" else: return "int32" def honour_petsc_dir_get_petsc_packages(): line = honour_petsc_dir_fetch_petscconf("#define PETSC_HAVE_PACKAGES") return set(line.split()[2].strip().strip(':"').split(':')) if mode == "install": # Handle command line arguments. parser = ArgumentParser(description="""Install firedrake and its dependencies.""", epilog="""The install process has three steps. 1. Any required system packages are installed using brew (MacOS) or apt (Ubuntu and similar Linux systems). On a Linux system without apt, the installation will fail if a dependency is not found. 2. A set of standard and/or third party Python packages is installed to the specified install location. 3. The core set of Python packages is downloaded to ./firedrake/src/ and installed to the specified location. The install creates a venv in ./firedrake (or in ./name where name is the parameter given to --venv-name) and installs inside that venv. The installer will ensure that the required configuration options are passed to PETSc. In addition, any configure options which you provide in the PETSC_CONFIGURE_OPTIONS environment variable will be honoured.""", formatter_class=RawDescriptionHelpFormatter) parser.add_argument("--slepc", action="store_true", help="Install SLEPc along with PETSc.") parser.add_argument("--opencascade", action="store_true", help="Install OpenCASCADE for CAD integration.") parser.add_argument("--tinyasm", action="store_true", help="Install TinyASM as backend for ASMPatchPC.") parser.add_argument("--torch", const="cpu", default=False, nargs='?', choices=["cpu", "cuda"], help="Install PyTorch for a CPU or CUDA backend (default: CPU).") parser.add_argument("--disable-ssh", action="store_true", help="Do not attempt to use ssh to clone git repositories: fall immediately back to https.") parser.add_argument("--no-package-manager", action='store_false', dest="package_manager", help="Do not attempt to use apt or homebrew to install operating system packages on which we depend.") group = parser.add_mutually_exclusive_group() group.add_argument("--minimal-petsc", action="store_true", help="Minimise the set of petsc dependencies installed. This creates faster build times (useful for build testing).") group.add_argument("--with-parmetis", action="store_true", help="Install PETSc with ParMETIS? (Non-free license, see http://glaros.dtc.umn.edu/gkhome/metis/parmetis/download)") parser.add_argument("--honour-petsc-dir", action="store_true", help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.") parser.add_argument("--petsc-int-type", choices=["int32", "int64"], default="int32", type=str, help="The integer type used by PETSc. Use int64 if you need to solve problems with more than 2 billion degrees of freedom. Only takes effect if firedrake-install builds PETSc.") parser.add_argument("--honour-pythonpath", action="store_true", help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.") parser.add_argument("--rebuild-script", action="store_true", help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.") parser.add_argument("--doi", type=str, nargs=1, help="Install a set of components matching a particular Zenodo DOI. The record should have been created with firedrake-zenodo.") # Used for testing if Zenodo broke the API # Tries to resolve to a known DOI and immediately exits. parser.add_argument("--test-doi-resolution", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--package-branch", type=str, nargs=2, action="append", metavar=("PACKAGE", "BRANCH"), help="Specify which branch of a package to use. This takes two arguments, the package name and the branch.") parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.") parser.add_argument("--mpicc", type=str, action="store", default=None, help="C compiler to use when building with MPI. If not set, MPICH will be downloaded and used.") parser.add_argument("--mpicxx", type=str, action="store", default=None, help="C++ compiler to use when building with MPI. If not set, MPICH will be downloaded and used.") parser.add_argument("--mpif90", type=str, action="store", default=None, help="Fortran compiler to use when building with MPI. If not set, MPICH will be downloaded and used.") parser.add_argument("--mpiexec", type=str, action="store", default=None, help="MPI launcher. If not set, MPICH will be downloaded and used.") parser.add_argument("--show-petsc-configure-options", action="store_true", help="Print out the configure options passed to PETSc and exit") parser.add_argument("--show-dependencies", action="store_true", help="Print out the package manager used and packages installed/required.") parser.add_argument("--venv-name", default="firedrake", type=str, action="store", help="Name of the venv to create and the name of the install directory relative to this install script (default is 'firedrake' with directory ./firedrake)") parser.add_argument("--install", action="append", dest="packages", help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir . Some additional packages have shortcut install options for more information see --install-help.") parser.add_argument("--pip-install", action="append", dest="pip_packages", help="Pip install additional packages into the venv") parser.add_argument("--install-help", action="store_true", help="Provide information on packages which can be installed using shortcut names.") parser.add_argument("--cache-dir", type=str, action="store", help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).") parser.add_argument("--complex", action="store_true", help="Installs the complex version of Firedrake; rebuilds PETSc in complex mode.") parser.add_argument("--documentation-dependencies", action="store_true", help="Install the dependencies required to build the documentation") parser.add_argument("--remove-build-files", action="store_true", help="Cleans up build artefacts from Firedrake venv. Unless you really cannot spare the disk space, this option is not recommended") parser.add_argument("--with-blas", default=("download" if osname == "Darwin" else None), help="Specify path to system BLAS directory. Use '--with-blas=download' to download openblas") parser.add_argument("--netgen", action="store_true", help="Install NGSolve/Netgen and ngsPETSc.") parser.add_argument("--no-vtk", action="store_true", help="Do not install VTK into the Firedrake virtualenv") args = parser.parse_args() # If the user has set any MPI info, they must set them all if args.mpicc or args.mpicxx or args.mpif90 or args.mpiexec: if not (args.mpicc and args.mpicxx and args.mpif90 and args.mpiexec): log.error("If you set any MPI information, you must set all of {mpicc, mpicxx, mpif90, mpiexec}.") sys.exit(1) if args.package_branch: branches = {package.lower(): branch for package, branch in args.package_branch} if args.test_doi_resolution: actual = resolve_doi_branches("10.5281/zenodo.1322546") expect = {'coffee': '87e50785d3a05b111f5423a66d461cd44cc4bdc9', 'finat': 'aa74fd499304c8363a4520555fd62ef21e8e5e1f', 'fiat': '184601a46c24fb5cbf8fd7961d22b16dd26890e7', 'firedrake': '6a30b64da01eb587dcc0e04e8e6b84fe4839bdb7', 'petsc': '413f72f04f5cb0a010c85e03ed029573ff6d4c63', 'petsc4py': 'ac2690070a80211dfdbab04634bdb3496c14ca0a', 'tsfc': 'fe9973eaacaa205fd491cd1cc9b3743b93a3d076', 'ufl': 'c5eb7fbe89c1091132479c081e6fa9c182191dcc', 'pyop2': '741a21ba9a62cb67c0aa300a2e199436ea8cb61c'} if actual != expect: log.error("Unable to resolve DOI correctly.") log.error("You'll need to figure out how Zenodo have changed their API.") sys.exit(1) else: log.info("DOI resolution test passed.") sys.exit(0) if args.doi: branches = resolve_doi_branches(args.doi[0]) if args.honour_petsc_dir: petsc_int_type = honour_petsc_dir_get_petsc_int_type() if petsc_int_type != args.petsc_int_type: petsc_dir, petsc_arch = honour_petsc_dir_get_petsc_dir() log.warning("Provided PETSc (PETSC_DIR=%s, PETSC_ARCH=%s) was compiled for %s, but given --petsc-int-type = %s: setting --petsc-int-type %s." % (petsc_dir, petsc_arch, petsc_int_type, args.petsc_int_type, petsc_int_type)) args.petsc_int_type = petsc_int_type args.prefix = False # Disabled as untested args.packages = args.packages or [] config = FiredrakeConfiguration(args) else: # This duplicates code from firedrake_configuration in order to avoid the module dependency and allow for installation recovery. try: with open(os.path.join(os.environ["VIRTUAL_ENV"], ".configuration.json"), "r") as f: config = json.load(f) except FileNotFoundError: # Fall back to the old location. import firedrake_configuration config = firedrake_configuration.get_config() if config is None: raise InstallError("Failed to find existing Firedrake configuration") parser = ArgumentParser(description="""Update this firedrake install to the latest versions of all packages.""", formatter_class=RawDescriptionHelpFormatter) parser.add_argument("--no-update-script", action="store_false", dest="update_script", help="Do not update script before updating Firedrake.") parser.add_argument("--rebuild", action="store_true", help="Rebuild all packages even if no new version is available. Usually petsc and petsc4py are only rebuilt if they change. All other packages are always rebuilt.") parser.add_argument("--rebuild-script", action="store_true", help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.") parser.add_argument("--slepc", action="store_true", dest="slepc", default=config["options"]["slepc"], help="Install SLEPc along with PETSc") parser.add_argument("--opencascade", action="store_true", dest="opencascade", default=config["options"].get("opencascade", False), help="Install OpenCASCADE for CAD integration.") parser.add_argument("--tinyasm", action="store_true", dest="tinyasm", default=config["options"].get("tinyasm", False), help="Install TinyASM as backend for ASMPatchPC.") parser.add_argument("--torch", const="cpu", nargs='?', choices=["cpu", "cuda"], default=config["options"].get("torch", False), help="Install PyTorch for a CPU or CUDA backend (default: CPU).") parser.add_argument("--honour-petsc-dir", action="store_true", default=config["options"]["honour_petsc_dir"], help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.") parser.add_argument("--petsc-int-type", choices=["int32", "int64"], default=config["options"]["petsc_int_type"], type=str, help="The integer type used by PETSc. Use int64 if you need to solve problems with more than 2 billion degrees of freedom. Only takes effect if firedrake-install builds PETSc.") parser.add_argument("--honour-pythonpath", action="store_true", default=config["options"].get("honour_pythonpath", False), help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.") parser.add_argument("--clean", action='store_true', help="Delete any remnants of obsolete Firedrake components.") parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.") parser.add_argument("--install", action="append", dest="packages", default=config["options"].get("packages", []), help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir. Some additional packages have shortcut install options for more information see --install-help.") parser.add_argument("--pip-install", action="append", dest="pip_packages", help="Pip install additional packages into the venv") parser.add_argument("--install-help", action="store_true", help="Provide information on packages which can be installed using shortcut names.") parser.add_argument("--cache-dir", type=str, action="store", default=config["options"].get("cache_dir", ""), help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).") complex_group = parser.add_mutually_exclusive_group() complex_group.add_argument("--complex", action="store_true", help="Updates to the complex version of Firedrake; rebuilds PETSc in complex mode.") complex_group.add_argument("--real", action="store_true", help="Updates to the real version of Firedrake; rebuilds PETSc in real mode.") parser.add_argument("--documentation-dependencies", action="store_true", help="Install the dependencies required to build the documentation") parser.add_argument("--show-dependencies", action="store_true", help="Print out the package manager used and packages installed/required.") parser.add_argument("--remove-build-files", action="store_true", help="Cleans up build artefacts from Firedrake venv. Unless you really cannot spare the disk space, this option is not recommended") parser.add_argument("--with-blas", default=("download" if osname == "Darwin" else None), help="Specify path to system BLAS directory. Use '--with-blas=download' to download openblas") parser.add_argument("--netgen", action="store_true", dest="netgen", default=config["options"].get("netgen", False), help="Install Netgen.") args = parser.parse_args() args.packages = list(set(args.packages)) # remove duplicates if args.honour_petsc_dir != config["options"]["honour_petsc_dir"]: log.error("You installed Firedrake with --honour-petsc-dir=%s, but are trying to update it with --honour-petsc-dir=%s." % (config["options"]["honour_petsc_dir"], args.honour_petsc_dir)) exit(1) if args.honour_petsc_dir: # Set int_type to the one found in petscconf.h petsc_int_type = honour_petsc_dir_get_petsc_int_type() if petsc_int_type != args.petsc_int_type: petsc_dir, petsc_arch = honour_petsc_dir_get_petsc_dir() log.warning("Provided PETSc (PETSC_DIR=%s, PETSC_ARCH=%s) was compiled for %s, but given --petsc-int-type = %s: setting --petsc-int-type %s." % (petsc_dir, petsc_arch, petsc_int_type, args.petsc_int_type, petsc_int_type)) args.petsc_int_type = petsc_int_type petsc_int_type_changed = False if config["options"]["petsc_int_type"] != args.petsc_int_type: petsc_int_type_changed = True args.rebuild = True if config["options"].get("with_blas") is not None: if config["options"].get("with_blas") != args.with_blas: # Rebuild if BLAS library explicitly changed at command line args.rebuild = True else: # Prevent deep_update overwriting BLAS options if they exist in the config, but are unspecified at the command line args.with_blas = config["options"].get("with_blas") petsc_complex_type_changed = False previously_complex = config["options"].get("complex", False) now_complex = not args.real and (args.complex or previously_complex) if previously_complex != now_complex: petsc_complex_type_changed = True args.rebuild = True args.complex = now_complex config = deep_update(config, FiredrakeConfiguration(args)) if args.install_help: help_string = """ You can install the following packages by passing --install shortname where shortname is one of the names given below: """ componentformat = "|{:10}|{:70}|\n" header = componentformat.format("Name", "Description") line = "-" * (len(header) - 1) + "\n" help_string += line + header + line for package, d in firedrake_apps.items(): help_string += componentformat.format(package, d[0]) help_string += line print(help_string) sys.exit(0) def sniff_compiler(exe): """Obtain the correct compiler class by calling the compiler executable. Attempt to determine the compiler version number. :arg cpp: If set to True will use the C++ compiler rather than the C compiler to determine the version number. :arg exe: String with name or path to compiler executable :returns: A compiler class """ # NAME try: output = subprocess.run( [exe, "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, encoding="utf-8" ).stdout except (subprocess.CalledProcessError, UnicodeDecodeError): output = "" # Find the name of the compiler family if output.startswith("gcc") or output.startswith("g++") or output.startswith("GNU"): name = "GNU" elif output.startswith("clang"): name = "clang" elif output.startswith("Apple LLVM") or output.startswith("Apple clang"): name = "clang" elif output.startswith("icc"): name = "Intel" elif "Cray" in output.split("\n")[0]: # Cray is more awkward eg: # Cray clang version 11.0.4 () # gcc (GCC) 9.3.0 20200312 (Cray Inc.) name = "Cray" else: name = "unknown" # VERSION version = None for dumpstring in ["-dumpfullversion", "-dumpversion"]: try: output = subprocess.run( [exe, dumpstring], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, encoding="utf-8" ).stdout version = Version(output) break except (subprocess.CalledProcessError, UnicodeDecodeError, InvalidVersion): continue return name, version # Temporary catch for tigerlake processors with gcc/g++/gfortran 9.3 if platform.system() == "Linux" and mode == "install": # Get the cpu information try: cpuinfostring = subprocess.run( ["lscpu"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, encoding="utf-8" ).stdout except (subprocess.CalledProcessError, UnicodeDecodeError): output = "" # Check if it is a tigerlake processor ie: "i3-11", "i5-11", "i7-11" if any(bad_cpu in cpuinfostring for bad_cpu in ["i3-11", "i5-11", "i7-11"]): # Get the compiler names and versions cc = sniff_compiler(args.mpicc or "gcc") cxx = sniff_compiler(args.mpicxx or "g++") fortran = sniff_compiler(args.mpif90 or "gfortran") # Check if the combination is a problem if any(map(lambda compiler: compiler[0] == "GNU" and compiler[1] < Version("9.4"), [cc, cxx, fortran])): print( "\nUsing a Tiger Lake (or newer) CPU with gcc, g++ and gfortran version<=9.3 is not possible." " Please install at least gcc, g++ and gfortran version 9.4.0 or later" ) sys.exit(1) @atexit.register def print_log_location(): log.info("\n\n%s log saved in %s" % (mode.capitalize(), logfile)) class directory(object): """Context manager that executes body in a given directory""" def __init__(self, dir): self.dir = os.path.abspath(dir) def __enter__(self): self.olddir = os.path.abspath(os.getcwd()) log.debug("Old path '%s'" % self.olddir) log.debug("Pushing path '%s'" % self.dir) os.chdir(self.dir) def __exit__(self, *args): log.debug("Popping path '%s'" % self.dir) os.chdir(self.olddir) log.debug("New path '%s'" % self.olddir) class environment(object): def __init__(self, **env): self.old = os.environ.copy() self.new = env def __enter__(self): os.environ.update(self.new) def __exit__(self, *args): os.environ = self.old options = config["options"] # Apply short cut package names. pyadjoint shoud no longer turn up the packages list because it is a hard dependency. options["packages"] = [firedrake_apps.get(p, (None, p))[1] for p in options["packages"] if not p.endswith("pyadjoint")] # Record of obsolete packages which --clean should remove from old installs. old_git_packages = ["dolfin-adjoint", "libadjoint"] if mode == "install": firedrake_env = os.path.abspath(args.venv_name) else: firedrake_env = os.environ["VIRTUAL_ENV"] if "cache_dir" not in config["options"] or not config["options"]["cache_dir"]: config["options"]["cache_dir"] = os.path.join(firedrake_env, ".cache") # venv install python = "%s/bin/python" % firedrake_env # Use the pip from the venv pip = [python, "-m", "pip"] pipinstall = pip + ["install", "--no-build-isolation", "--no-binary", ",".join(wheel_blacklist)] if mode == "install": # "petsc4py" is in `branches` when we are explicitly asking for a petsc4py # branch that lives in firedrake-project/petsc4py. This happens when we do: # python3 firedrake-install --doi some_old_zenodo_doi ... # or: # python3 firedrake-install --package-branch petsc4py some_old_branch ... use_petsc4py_in_petsc = "petsc4py" not in branches use_slepc4py_in_slepc = "slepc4py" not in branches else: use_petsc4py_in_petsc = True use_slepc4py_in_slepc = True # This context manager should be used whenever arguments should be temporarily added to pipinstall. class pipargs(object): def __init__(self, *args): self.args = args def __enter__(self): self.save = pipinstall.copy() pipinstall.extend(self.args) def __exit__(self, *args, **kwargs): global pipinstall pipinstall = self.save def check_call(arguments): try: log.debug("Running command '%s'", " ".join(arguments)) log.debug(subprocess.check_output(arguments, stderr=subprocess.STDOUT, env=os.environ).decode()) except subprocess.CalledProcessError as e: log.debug(e.output.decode()) raise def check_output(args): try: log.debug("Running command '%s'", " ".join(args)) return subprocess.check_output(args, stderr=subprocess.STDOUT, env=os.environ).decode() except subprocess.CalledProcessError as e: log.debug(e.output.decode()) raise pyinstall = [python, "setup.py", "install"] if "PYTHONPATH" in os.environ and not args.honour_pythonpath: quit("""The PYTHONPATH environment variable is set. This is probably an error. If you really want to use your own Python packages, please run again with the --honour-pythonpath option. """) def brew_gcc_libdir(): brew_gcc_prefix = check_output(["brew", "--prefix", "gcc"])[:-1] brew_gcc_info = json.loads(check_output(["brew", "info", "--json", "gcc"])[:-1]) gcc_major_version = brew_gcc_info[0]["installed"][0]["version"].split(".")[0] return brew_gcc_prefix + "/lib/gcc/" + gcc_major_version # See .github/workflows/docker.yml to see the PETSc options used on CI. # Edit these as appropriate if PETSc external packages are added or removed. def get_minimal_petsc_packages(): pkgs = set() # File format pkgs.add("hdf5") # Sparse direct solver pkgs.add("superlu_dist") # Parallel mesh partitioner pkgs.add("ptscotch") # Sparse direct solver pkgs.add("scalapack") # Needed for mumps pkgs.add("mumps") if not options["complex"]: # AMG pkgs.add("hypre") if options["petsc_int_type"] == "int32": # Serial mesh partitioner pkgs.add("chaco") return pkgs def get_petsc_options(minimal=False): # The logic in this function is getting out of hand... petsc_options = {"--with-fortran-bindings=0", "--with-debugging=0", "--with-shared-libraries=1", "--with-c2html=0", # Parser generator "--download-bison"} for pkg in get_minimal_petsc_packages(): petsc_options.add("--download-" + pkg) if osname == "Darwin": petsc_options.add("--with-x=0") # These three lines are used to inspect the MacOS Command Line Tools (CLT) version cmd = ["pkgutil", "--pkg-info=com.apple.pkg.CLTools_Executables"] output = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="UTF-8") clt = {k.strip(): v.strip() for k, v in [line.split(':') for line in output.stdout.split('\n') if line]} elif osname == "Linux": # PETSc requires cmake version 3.18.1 or higher. petsc_options.add("--download-cmake") if (not options["minimal_petsc"]) and (not minimal): petsc_options.add("--with-zlib") # File formats petsc_options.add("--download-netcdf") petsc_options.add("--download-pnetcdf") # Sparse direct solvers petsc_options.add("--download-suitesparse") petsc_options.add("--download-pastix") # Required by pastix petsc_options.add("--download-hwloc") if osname == "Darwin": petsc_options.add("--download-hwloc-configure-arguments=--disable-opencl") # Serial mesh partitioner petsc_options.add("--download-metis") if options.get("with_parmetis", None): # Non-free license. petsc_options.add("--download-parmetis") if options["petsc_int_type"] != "int32": petsc_options.add("--with-64-bit-indices") if options["complex"]: petsc_options.add('--with-scalar-type=complex') if options.get("mpiexec") is not None: petsc_options.add("--with-mpiexec={}".format(options["mpiexec"])) petsc_options.add("--with-cc={}".format(options["mpicc"])) petsc_options.add("--with-cxx={}".format(options["mpicxx"])) petsc_options.add("--with-fc={}".format(options["mpif90"])) else: # Download mpich if the user does not tell us about an MPI. petsc_options.add("--download-mpich") if osname == "Darwin": petsc_options.add("--download-mpich-configure-arguments=--disable-opencl") if options.get("with_blas") == "download": # Download openblas if the user specifies. petsc_options.add("--download-openblas") petsc_options.add("--download-openblas-make-options='USE_THREAD=0 USE_LOCKING=1 USE_OPENMP=0'") if osname == "Darwin": petsc_options.add("--CFLAGS=-Wno-implicit-function-declaration") if Version(clt["version"]) >= Version("15"): # CLT >= 15 requires legacy linking behaviour petsc_options.add("--LDFLAGS=-Wl,-ld_classic") elif options.get("with_blas") is not None: petsc_options.add("--with-blaslapack-dir={}".format(options["with_blas"])) if osname == "Darwin": petsc_options.add("--CFLAGS=-I{}/include -Wno-implicit-function-declaration".format(options["with_blas"])) if Version(clt["version"]) >= Version("15"): # CLT >= 15 requires legacy linking behaviour petsc_options.add("--LDFLAGS=-Wl,-ld_classic,-rpath,{0}/lib -L{0}/lib".format(options["with_blas"])) else: petsc_options.add("--CFLAGS=-I{}/include".format(options["with_blas"])) petsc_options.add("--LDFLAGS=-Wl,-rpath,{0}/lib -L{0}/lib".format(options["with_blas"])) elif osname == "Darwin": brew_blas_prefix = check_output(["brew", "--prefix", "openblas"])[:-1] petsc_options.add("--with-blaslapack-dir={}".format(brew_blas_prefix)) petsc_options.add("--CFLAGS=-I{}/include -Wno-implicit-function-declaration".format(brew_blas_prefix)) if Version(clt["version"]) >= Version("15"): # CLT >= 15 requires legacy linking behaviour petsc_options.add("--LDFLAGS=-Wl,-ld_classic,-rpath,{0}/lib -L{0}/lib -L{1}".format(brew_blas_prefix, brew_gcc_libdir())) else: petsc_options.add("--LDFLAGS=-Wl,-rpath,{0}/lib -L{0}/lib -L{1}".format(brew_blas_prefix, brew_gcc_libdir())) if not minimal: for option in shlex.split(os.environ.get("PETSC_CONFIGURE_OPTIONS", "")): if option.startswith("--with-hdf5-dir"): petsc_options.discard("--download-hdf5") os.environ["HDF5_DIR"] = option.replace("--with-hdf5-dir=", "") petsc_options.add(option) if "HDF5_DIR" in os.environ and "--download-hdf5" in petsc_options: del os.environ["HDF5_DIR"] log.info("\nWarning: HDF5_DIR environment variable set, but ignored; " "use PETSC_CONFIGURE_OPTIONS=\"--with-hdf5-dir=$HDF5_DIR\" " "to have PETSc built against $HDF5_DIR.") return list(petsc_options) def get_petsc_config(): petsc_dir, petsc_arch = get_petsc_dir() with open(os.path.join(petsc_dir, petsc_arch, "include", "petscconfiginfo.h")) as fh: for line in fh.readlines(): if line.startswith("static const char *petscconfigureoptions"): info = line.split("=", maxsplit=1)[1].strip(' ";\n').split('--')[1:] break return ["--" + arg for arg in info] if mode == "update" and petsc_int_type_changed: log.warning("""Force rebuilding all packages because PETSc int type changed""") if mode == "update" and petsc_complex_type_changed: log.warning("""Force rebuilding all packages because PETSc scalar type changed between real and complex""") def brew_install(name, options=None): arguments = [name] if options: arguments = options + arguments if args.verbose: arguments = ["--verbose"] + arguments check_call(["brew", "install"] + arguments) def apt_check(name): log.info("Checking for presence of package %s..." % name) # Note that subprocess return codes have the opposite logical # meanings to those of Python variables. try: check_call(["dpkg-query", "-s", name]) log.info(" installed.") return True except subprocess.CalledProcessError: log.info(" missing.") return False def apt_install(names): log.info("Installing missing packages: %s." % ", ".join(names)) if sys.stdin.isatty(): subprocess.check_call(["sudo", "apt-get", "install"] + names) else: log.info("Non-interactive stdin detected; installing without prompts") subprocess.check_call(["sudo", "apt-get", "-y", "install"] + names) def split_requirements_url(url): name = url.split(".git")[0].split("#")[0].split("/")[-1] spliturl = url.split("://")[1].split("#")[0].split("@") try: plain_url, branch = spliturl except ValueError: plain_url = spliturl[0] branch = "master" return name, plain_url, branch def git_url(plain_url, protocol): if protocol == "ssh": return "git@%s:%s" % tuple(plain_url.split("/", 1)) elif protocol == "https": return "https://%s" % plain_url else: raise ValueError("Unknown git protocol: %s" % protocol) def git_clone(url): name, plain_url, branch = split_requirements_url(url) if name == "petsc" and args.honour_petsc_dir: log.info("Using existing PETSc installation\n") return name elif name == "petsc4py" and use_petsc4py_in_petsc: log.info("Not cloning petsc4py from firedrake-project repo: using petsc4py in PETSc source tree.\n") return name log.info("Cloning %s\n" % name) branch = branches.get(name.lower(), branch) try: if options["disable_ssh"]: raise InstallError("Skipping ssh clone because --disable-ssh") # note: so far only loopy requires submodule check_call(["git", "clone", "-q", "--recursive", git_url(plain_url, "ssh")]) log.info("Successfully cloned repository %s" % name) except (subprocess.CalledProcessError, InstallError): if not options["disable_ssh"]: log.warning("Failed to clone %s using ssh, falling back to https." % name) try: check_call(["git", "clone", "-q", "--recursive", git_url(plain_url, "https")]) log.info("Successfully cloned repository %s." % name) except subprocess.CalledProcessError: log.error("Failed to clone %s branch %s." % (name, branch)) raise with directory(name): try: log.info("Checking out branch %s" % branch) check_call(["git", "checkout", "-q", branch]) log.info("Successfully checked out branch %s" % branch) except subprocess.CalledProcessError: log.error("Failed to check out branch %s" % branch) raise try: log.info("Updating submodules.") check_call(["git", "submodule", "update", "--recursive"]) log.info("Successfully updated submodules.") except subprocess.CalledProcessError: log.error("Failed to update submodules.") raise return name def list_cloned_dependencies(name): log.info("Finding dependencies of %s\n" % name) deps = OrderedDict() try: for dep in open(name + "/requirements-git.txt", "r"): name = split_requirements_url(dep.strip())[0] deps[name] = dep.strip() except IOError: pass return deps def clone_dependencies(name): log.info("Cloning the dependencies of %s" % name) deps = [] try: for dep in open(name + "/requirements-git.txt", "r"): deps.append(git_clone(dep.strip())) except IOError: pass return deps def git_update(name, url=None): # Update the named git repo and return true if the current branch actually changed. log.info("Updating the git repository for %s" % name) with directory(name): git_sha = check_output(["git", "rev-parse", "HEAD"]) # Ensure remotes get updated if and when we move repositories. if url: _, plain_url, branch = split_requirements_url(url) current_url = check_output(["git", "remote", "-v"]).split()[1] current_branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() protocol = "https" if current_url.startswith("https") else "ssh" new_url = git_url(plain_url, protocol) # Ensure we only change from bitbucket to github and not the reverse. if (new_url != current_url and ("bitbucket.org" in current_url) and ("github.com/firedrakeproject" in plain_url or "github.com/dolfin-adjoint" in plain_url)): log.info("Updating git remote for %s" % name) check_call(["git", "remote", "set-url", "origin", new_url]) # Ensure we only switch loopy branch if loopy is on firedrake and not on a feature branch. elif name == "loopy" and current_branch != branch and current_branch == "firedrake": log.info("Updating loopy branch to main") check_call(["git", "checkout", "-q", branch]) check_call(["git", "pull", "--recurse-submodules"]) git_sha_new = check_output(["git", "rev-parse", "HEAD"]) return git_sha != git_sha_new def run_pip(args): check_call(pip + args) def run_pip_install(pipargs): # Make pip verbose when logging, so we see what the # subprocesses wrote out. # Particularly important for debugging petsc fails. with environment(**blas): pipargs = ["-vvv"] + pipargs check_call(pipinstall + pipargs) def run_cmd(args): check_call(args) def get_requirements(reqfname): with open(reqfname, "r") as f: reqs = f.readlines() return [req.strip() for req in reqs] def run_pip_install_wrap(reqs, parallel_compiler_env): for req in reqs: package_name = re.split('<|>|=', req)[0] if package_name in parallel_packages: with environment(**parallel_compiler_env): run_pip_install([req]) elif package_name == "numpy": # Downgrade setuptools and wheel for numpy run_pip(["install", "-U", "setuptools==59.2.0"]) run_pip(["install", "-U", "wheel==0.37.0"]) run_pip_install(["numpy==1.24"]) # Upgrade setuptools and wheel for everything else run_pip(["install", "-U", "setuptools"]) run_pip(["install", "-U", "wheel"]) else: run_pip_install([req]) def pip_requirements(package, parallel_compiler_env): log.info("Installing pip dependencies for %s" % package) if os.path.isfile("%s/requirements-ext.txt" % package): reqs = get_requirements("%s/requirements-ext.txt" % package) run_pip_install_wrap(reqs, parallel_compiler_env) elif os.path.isfile("%s/requirements.txt" % package): if package == "COFFEE": # FIXME: Horrible hack to work around # https://github.com/coin-or/pulp/issues/123 run_pip_install(["--no-deps", "-r", "%s/requirements.txt" % package]) elif package == "pyadjoint": reqs = get_requirements("%s/requirements.txt" % package) for req in reqs: try: run_pip_install([req]) except subprocess.CalledProcessError as e: if req == "tensorflow": log.debug("Skipping failing install of optional pyadjoint dependency tensorflow") else: raise e elif package in {"loopy", "icepack"}: # dependencies are installed in setup.py return else: reqs = get_requirements("%s/requirements.txt" % package) run_pip_install_wrap(reqs, parallel_compiler_env) else: log.info("No dependencies found. Skipping.") def install(package): log.info("Installing %s" % package) if package == "petsc/": build_and_install_petsc() elif package == "slepc/": build_and_install_slepc() elif package == "petsc4py/": build_and_install_x4py("petsc4py") elif package == "slepc4py/": build_and_install_x4py("slepc4py") elif package == "h5py/": # h5py tries to build against ancient version of numpy additional flag avoids this run_pip_install(["--no-build-isolation", package]) elif package == "loopy/" and osname == "Darwin": # On MacOS loopy won't build unless additional flags are passed to the compiler with environment(CFLAGS="-Wno-implicit-function-declaration"): run_pip_install(["-e", package]) else: run_pip_install(["-e", package]) def clean(package): log.info("Cleaning %s" % package) with directory(package): with environment(**blas): # loopy needs knowledge of the system BLAS just to run setup.py if os.path.isfile("setup.py"): check_call([python, "setup.py", "clean"]) else: # PEP 518 build system if os.path.exists("build") and os.path.isdir("build"): shutil.rmtree("build") def pip_uninstall(package): log.info("Removing existing %s installations\n" % package) # Uninstalling something with pip is an absolute disaster. We # have to use pip freeze to list all available packages "locally" # and keep on removing the one we want until it is gone from this # list! Yes, pip will happily have two different versions of the # same package co-existing. Moreover, depending on the phase of # the moon, the order in which they are uninstalled is not the # same as the order in which they appear on sys.path! again = True uninstalled = False i = 0 while again: # List installed packages, "locally". In a venv, # this just tells me packages in the venv, otherwise it # gives me everything. lines = check_output(pip + ["freeze", "-l"]) again = False for line in lines.split("\n"): # Do we have a locally installed package? if line.startswith(package+'=='): # Uninstall it. run_pip(["uninstall", "-y", line.strip()]) uninstalled = True # Go round again, because THERE MIGHT BE ANOTHER ONE! again = True i += 1 if i > 10: raise InstallError("pip claims it uninstalled %s more than 10 times. Something is probably broken.", package) return uninstalled def get_petsc_dir(): if args.honour_petsc_dir: return honour_petsc_dir_get_petsc_dir() else: petsc_dir = os.path.join(os.environ["VIRTUAL_ENV"], "src", "petsc") petsc_arch = "default" return petsc_dir, petsc_arch def get_petsc4py_dir(): if use_petsc4py_in_petsc: petsc_dir, _ = get_petsc_dir() # petsc4py is currently here: return os.path.join(petsc_dir, "src/binding/petsc4py/") else: return "petsc4py/" def get_slepc_dir(): petsc_dir, petsc_arch = get_petsc_dir() if args.honour_petsc_dir: try: slepc_dir = os.environ["SLEPC_DIR"] except KeyError: raise InstallError("Need to set SLEPC_DIR for --slepc with --honour-petsc-dir") else: slepc_dir = os.path.join(os.environ["VIRTUAL_ENV"], "src", "slepc") return slepc_dir, petsc_arch def get_slepc4py_dir(): if use_slepc4py_in_slepc: slepc_dir, _ = get_slepc_dir() return os.path.join(slepc_dir, "src/binding/slepc4py") else: return "slepc4py/" def build_and_install_petsc(): log.info("Depending on your platform, PETSc may take an hour or more to build!") assert not args.honour_petsc_dir # Belt and braces security before we start clobbering things. with directory("petsc"): petsc_dir, petsc_arch = get_petsc_dir() check_call(["rm", "-rf", os.path.join(petsc_dir, petsc_arch)]) petsc_options = get_petsc_options() # Set MACOSX_DEPLOYMENT_TARGET to the current macos version # to workaround issues configuring PETSc on macos. # Choosing 12.0 as the oldest MacOS version supported as of 2024-02-08 macosx_deployment_target = os.environ.get("MACOSX_DEPLOYMENT_TARGET", "12.0") with environment(MACOSX_DEPLOYMENT_TARGET=macosx_deployment_target): check_call([python, "./configure", "PETSC_DIR={}".format(petsc_dir), "PETSC_ARCH={}".format(petsc_arch)] + petsc_options) check_call(["make", "PETSC_DIR={}".format(petsc_dir), "PETSC_ARCH={}".format(petsc_arch), "all"]) if args.remove_build_files and not args.honour_petsc_dir: log.info("Cleaning up PETSc build artifacts") check_call(["rm", "-rf", os.path.join(petsc_dir, petsc_arch, "externalpackages")]) check_call(["rm", "-rf", os.path.join(petsc_dir, "src", "docs")]) tutorial_output = os.path.join(petsc_dir, "src", "**", "tutorials", "output", "*") test_output = os.path.join(petsc_dir, "src", "**", "tests", "output", "*") for deletefile in chain(iglob(tutorial_output, recursive=True), iglob(test_output, recursive=True)): check_call(["rm", "-f", deletefile]) def build_and_install_x4py(package="petsc4py"): if package == "petsc4py": x4py_dir = get_petsc4py_dir() elif package == "slepc4py": x4py_dir = get_slepc4py_dir() else: raise ValueError("build_and_install_x4py() should be called with 'petsc4py' or 'slepc4py' only") with environment(**blas): if args.honour_petsc_dir: log.info("Copying {} to virtualenv to install".format(package)) x4py_copy = os.path.join(os.environ['VIRTUAL_ENV'], 'src', package) if not os.path.exists(x4py_copy): shutil.copytree(x4py_dir, x4py_copy, dirs_exist_ok=True) pipargs = ["-vvv", "--ignore-installed", x4py_copy] try: check_output(pipinstall + pipargs) except subprocess.CalledProcessError: with open(os.path.join(x4py_copy, "README.md")) as fh: fh.write("# Unable to install {}\n".format(package)) fh.write("This is just a copy of the {} source for debugging\n".format(package)) fh.write("See `firedrake-install.log` for the error") raise else: shutil.rmtree(x4py_copy) else: pipargs = ["-vvv", "--ignore-installed", x4py_dir] check_output(pipinstall + pipargs) def build_and_install_h5py(): log.info("Installing h5py") # Clean up old downloads if os.path.exists("h5py-2.5.0"): log.info("Removing old h5py-2.5.0 source") shutil.rmtree("h5py-2.5.0") if os.path.exists("h5py.tar.gz"): log.info("Removing old h5py.tar.gz") os.remove("h5py.tar.gz") url = "git+https://github.com/firedrakeproject/h5py.git@firedrake#egg=h5py" if os.path.exists("h5py"): changed = False with directory("h5py"): # Rewrite old h5py/h5py remote to firedrakeproject/h5py remote. plain_url = split_requirements_url(url)[1] current_remote = check_output(["git", "remote", "-v"]).split()[1] proto = "https" if current_remote.startswith("https") else "ssh" new_remote = git_url(plain_url, proto) if new_remote != current_remote and "h5py/h5py.git" in current_remote: log.info("Updating git remote for h5py from %s to %s", current_remote, new_remote) check_call(["git", "remote", "set-url", "origin", new_remote]) check_call(["git", "fetch", "-p", "origin"]) check_call(["git", "checkout", "firedrake"]) changed = True changed |= git_update("h5py") # If h5py installation fails for some reason, we end up having a # situation where h5py is not installed, but h5py directory is # up-to-date, so we need to handle this here. changed |= not is_installed("h5py") else: git_clone(url) changed = True petsc_dir, petsc_arch = get_petsc_dir() hdf5_dir = os.environ.get("HDF5_DIR", "%s/%s" % (petsc_dir, petsc_arch)) if changed or args.rebuild: log.info("Linking h5py against HDF5_DIR=%s\n" % hdf5_dir) with environment(HDF5_DIR=hdf5_dir, HDF5_MPI="ON", **blas): # Only uninstall if things changed. pip_uninstall("h5py") # Pip installing from dirty directory is potentially unsafe. with directory("h5py/"): check_call(["git", "clean", "-fdx"]) install("h5py/") else: log.info("No need to rebuild h5py") def build_and_install_libsupermesh(cc, cxx, f90, mpiexec): log.info("Installing libsupermesh") url = "git+https://github.com/firedrakeproject/libsupermesh.git" if os.path.exists("libsupermesh"): changed = git_update("libsupermesh", url) else: git_clone(url) changed = True if changed: with directory("libsupermesh"): check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "-f", "-x", "-d"]) check_call(["mkdir", "-p", "build"]) with directory("build"): cmd = [ "cmake", "..", "-DBUILD_SHARED_LIBS=ON", "-DCMAKE_INSTALL_PREFIX=" + firedrake_env, "-DMPI_C_COMPILER=" + cc, "-DMPI_CXX_COMPILER=" + cxx, "-DMPI_Fortran_COMPILER=" + f90, "-DCMAKE_Fortran_COMPILER=" + f90, "-DMPIEXEC_EXECUTABLE=" + mpiexec, ] check_call(cmd) check_call(["make"]) check_call(["make", "install"]) else: log.info("No need to rebuild libsupermesh") def build_and_install_tinyasm(): log.info("Installing TinyASM") if os.path.exists("TinyASM"): log.info("Updating the git repository for TinyASM") with directory("TinyASM"): check_call(["git", "fetch"]) git_sha = check_output(["git", "rev-parse", "HEAD"]) git_sha_new = check_output(["git", "rev-parse", "@{u}"]) tinyasm_changed = git_sha != git_sha_new if tinyasm_changed: check_call(["git", "reset", "--hard"]) check_call(["git", "pull"]) else: git_clone("git+https://github.com/florianwechsung/TinyASM.git") tinyasm_changed = True if tinyasm_changed: with directory("TinyASM"): # Clean source directory check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "-f", "-x", "-d"]) run_pip_install(["-e", "."]) else: log.info("No need to rebuild TinyASM") def build_and_install_pythonocc(): log.info("Installing pythonocc-core") url = "git+https://github.com/tpaviot/pythonocc-core.git@595b0a4e8e60e8d6011bea0cdb54ac878efcfcd2" if not os.path.exists("pythonocc-core"): git_clone(url) with directory("pythonocc-core"): check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "-f", "-x", "-d"]) check_call(["mkdir", "-p", "build"]) with directory("build"): check_call(["cmake", "..", "-DCMAKE_INSTALL_PREFIX=" + firedrake_env, "-DPYTHON_EXECUTABLE=" + python, "-DMPI_C_COMPILER=" + cc, "-DMPI_CXX_COMPILER=" + cxx, "-DMPI_Fortran_COMPILER=" + f90, "-DCMAKE_Fortran_COMPILER=" + f90]) check_call(["make", "install"]) else: log.info("No need to rebuild pythonocc-core") def build_and_install_torch(): """Install PyTorch for a CPU or CUDA backend.""" log.info("Installing PyTorch (backend: %s)" % args.torch) if osname == "Darwin" and args.torch == "cuda": raise InstallError("CUDA installation is not available on MacOS.") if sys.version_info >= (3, 10): run_pip_install(["typing_extensions"]) extra_index_url = ["--extra-index-url", "https://download.pytorch.org/whl/cpu"] if args.torch == "cpu" else [] run_pip_install(["torch"] + extra_index_url) def build_and_install_slepc(): petsc_dir, petsc_arch = get_petsc_dir() slepc_dir, slepc_arch = get_slepc_dir() assert petsc_arch == slepc_arch slepc_changed = False if args.honour_petsc_dir: log.info("Using installed SLEPc from %s/%s", slepc_dir, petsc_arch) else: log.info("Installing SLEPc.") url = "git+https://github.com/firedrakeproject/slepc.git@firedrake" if os.path.exists("slepc"): slepc_changed |= git_update("slepc") else: git_clone(url) slepc_changed = True if slepc_changed: with directory("slepc"): check_call(["rm", "-rf", os.path.join(slepc_dir, petsc_arch)]) with environment(PETSC_DIR=petsc_dir, SLEPC_DIR=slepc_dir, PETSC_ARCH=petsc_arch): check_call([python, "./configure"]) check_call(["make", "all"]) check_call(["make", "check"]) else: log.info("No need to rebuild SLEPc") url = "git+https://github.com/firedrakeproject/slepc4py.git@firedrake" if not use_slepc4py_in_slepc: if os.path.exists("slepc4py"): with directory("slepc4py"): plain_url = split_requirements_url(url)[1] current_remote = check_output(["git", "remote", "-v"]).split()[1] proto = "https" if current_remote.startswith("https") else "ssh" new_remote = git_url(plain_url, proto) if new_remote != current_remote: log.info("Updating git remote for slepc4py from %s to %s", current_remote, new_remote) check_call(["git", "remote", "set-url", "origin", new_remote]) check_call(["git", "fetch", "-p", "origin"]) check_call(["git", "checkout", "firedrake"]) else: log.info("Cloning slepc4py from %s" % url) git_clone(url) def build_and_install_netgen(): log.info("Installing NGSolve/Netgen.") run_pip(["install", "netgen-mesher"]) log.info("Installing ngsPETSc.") url = "git+https://github.com/NGSolve/ngsPETSc.git@main" ngsPETSc_changed = False if os.path.exists("ngsPETSc"): ngsPETSc_changed |= git_update("ngsPETSc") else: git_clone(url) ngsPETSc_changed = True if ngsPETSc_changed: with directory("ngsPETSc"): with environment(NGSPETSC_NO_INSTALL_REQUIRED="ON"): check_call([python, "-m", "pip", "install", "-e", "."]) def install_documentation_dependencies(): """Just install the required dependencies. There are no provenance issues here so no need to record this in the configuration dict.""" log.info("Installing documentation dependencies") run_pip(["install", "-U", "sphinx"]) run_pip(["install", "-U", "sphinx-autobuild"]) run_pip(["install", "-U", "sphinxcontrib-bibtex"]) run_pip(["install", "-U", "sphinxcontrib-svg2pdfconverter"]) run_pip(["install", "-U", "sphinxcontrib-jquery"]) run_pip(["install", "-U", "bibtexparser"]) run_pip(["install", "-U", "sphinxcontrib-youtube"]) run_pip(["install", "-U", "numpydoc"]) def clean_obsolete_packages(): dirs = os.listdir(".") for package in old_git_packages: pip_uninstall(package) if package in dirs: shutil.rmtree(package) def quit(message): log.error(message) sys.exit(1) def build_update_script(): log.info("Creating firedrake-update script.") with open("firedrake/scripts/firedrake-install", "r") as f: update_script = f.read() try: os.mkdir("../bin") except OSError: pass with open("../bin/firedrake-update", "w") as f: f.write(update_script) check_call(["chmod", "a+x", "../bin/firedrake-update"]) if args.rebuild_script: os.chdir(os.path.dirname(os.path.realpath(__file__)) + ("/../..")) build_update_script() log.info("Successfully rebuilt firedrake-update.\n") log.info("To upgrade firedrake, run firedrake-update") sys.exit(0) def create_compiler_env(cc, cxx, f90): env = dict() if cc: env["MPICC"] = cc env["MPI_C_COMPILER"] = cc env["CC"] = cc # Work around homebrew adding /usr/local/lib to the library search path. if osname == "Darwin": env["CFLAGS"] = " ".join(( os.environ.get("CFLAGS", ""), "-L" + os.path.join(*get_petsc_dir(), "lib"), "-I" + os.path.join(*get_petsc_dir(), "include") )) if cxx: env["MPICXX"] = cxx env["MPI_CXX_COMPILER"] = cxx env["CXX"] = cxx if f90: env["MPIF90"] = f90 env["MPI_C_COMPILER"] = f90 env["F90"] = f90 return env if mode == "install" and args.show_petsc_configure_options: log.info("*********************************************") log.info("Would build PETSc with the following options:") log.info("*********************************************\n") petsc_options = get_petsc_options() log.info("\n".join(petsc_options)) sys.exit(0) if "PETSC_DIR" in os.environ and not args.honour_petsc_dir: quit("""The PETSC_DIR environment variable is set. This is probably an error. If you really want to use your own PETSc build, please run again with the --honour-petsc-dir option. """) if "PETSC_DIR" not in os.environ and args.honour_petsc_dir: quit("""The --honour-petsc-dir is set, but PETSC_DIR environment variable is not defined. If you have compiled PETSc manually, set PETSC_DIR (and optionally PETSC_ARCH) variables to point to the build directory. """) if "SLEPC_DIR" not in os.environ and args.honour_petsc_dir and options["slepc"]: quit("""If you use --honour-petsc-dir, you must also build SLEPc manually and set the SLEPC_DIR environment variable appropriately""") if "SLEPC_DIR" in os.environ and not args.honour_petsc_dir and options["slepc"]: quit("""The SLEPC_DIR environment variable is set. If you want to use your own SLEPc version, you must also build your own PETSc and run with --honour-petsc-dir.""") if platform.uname()[0] == "Darwin" and args.opencascade: quit("""Sorry, automatically installing opencascade on OSX hasn't been implemented yet. (It's not a supported package in brew.) Please contact us to get this working.""") log.debug("Installer running with: %s" % sys.executable) log.debug("Python version: %s" % sys.version) log.debug("*** Current environment (output of 'env') ***") log.debug(check_output(["env"])) log.debug("\n\n") if mode == "install" and args.honour_petsc_dir: minimal_packages = get_minimal_petsc_packages() minimal_packages.update({"blaslapack", "mathlib", "mpi"}) current_packages = honour_petsc_dir_get_petsc_packages() log.debug("*****************************************************") log.debug("Specified PETSc was built with the following options:") log.debug("*****************************************************\n") log.debug("PETSC_DIR={} PETSC_ARCH={}".format(*get_petsc_dir())) log.debug("\n".join(sorted(get_petsc_config())) + '\n') log.debug("The following external packages were detected and are used by PETSc:") log.debug(", ".join(sorted(current_packages))) log.debug("\n") if not minimal_packages < current_packages: log.error("*********************************************************************") log.error("Specified PETSc is missing the following packages:") log.error(", ".join(sorted(minimal_packages - current_packages)) + '\n') log.error("You must ensure that PETSc has been built with these minimal options:") log.error("*********************************************************************\n") log.error("\n".join(sorted(get_petsc_options(minimal=True)))) log.error("") log.error('Use --show-petsc-configure-options as an argument to the firedrake-install script to display options if additional functionality is required.\n ') log.error("FATAL: Unable to install with specified PETSc") exit(1) if mode == "install" or not args.update_script: # Check operating system. required_packages = None package_manager = None if osname == "Darwin": package_manager = "brew" required_packages = ["gcc", "autoconf", "make", "automake", "cmake", "libtool", "boost"] if args.with_blas is None and mode == "install": required_packages.append("openblas") elif osname == "Linux": package_manager = "apt-get" required_packages = ["build-essential", "autoconf", "automake", "bison", # for ptscotch "flex", # for ptscotch "cmake", "gfortran", "git", "libcurl4-openssl-dev", "pkgconf", # for p4est "libtool", "libxml2-dev", "python3-dev", "python3-pip", "python3-tk", "python3-venv", "zlib1g-dev", "libboost-dev"] # For ROL in pyadjoint if args.opencascade: required_packages.append("liboce-ocaf-dev") required_packages.append("swig") if args.with_blas is None and mode == "install": required_packages.append("libopenblas-dev") else: log.warning("You do not appear to be running Linux or MacOS. Please do not be surprised if this install fails.") if args.documentation_dependencies: required_packages.append("graphviz") if args.show_dependencies: log.info("\n************************************************") log.info("The following dependencies need to be installed:") log.info("************************************************\n") log.info("* A C and C++ compiler (for example gcc/g++ or clang), GNU make") log.info("* A Fortran compiler (for PETSc)") log.info("* Blas and Lapack") log.info("* Boost") log.info("* Git") log.info("* Python version >=3.6") log.info("* The Python headers") log.info("* autoconf, automake, libtool") log.info("* CMake") if osname == "Linux": log.info("* zlib, bison, flex") if package_manager: log.info("\n********************************************") log.info("The following %s packages are recommended:" % (package_manager)) log.info("********************************************\n") log.info("\n".join(required_packages)) log.info("\nThey will be installed by %s unless --no-package-manager is used." % (package_manager)) if osname == "Darwin": log.info("\nIn addition, the command line tools will be installed using xcode") sys.exit(0) blas_location = args.with_blas if osname == "Darwin": try: check_call(["brew", "--version"]) except subprocess.CalledProcessError: quit("Installing Firedrake on MacOS requires Homebrew. Please install " "it using the instructions at http://brew.sh and then try again.") if options["package_manager"]: log.info("Installing command line tools...") try: check_call(["xcode-select", "--install"]) except subprocess.CalledProcessError: # expected failure if already installed pass log.info("Checking brew doctor...") try: check_call(["brew", "doctor"]) except subprocess.CalledProcessError: pass log.info("Installing required packages via homebrew...") # Ensure a fortran compiler is available installed_packages = dict((p["name"], p) for p in json.loads(check_output(["brew", "info", "--installed", "--json"]))) # Handle homebrew migration of default python == python3 if installed_packages.get("python", {}).get("oldname") == "python3": installed_packages["python3"] = installed_packages["python"] for package in required_packages: if package in installed_packages: log.info("Required package '{0}' is already installed via Homebrew.".format(package)) log.debug(pprint.pformat(installed_packages[package])) else: log.info("Installing required package '{0}' via Homebrew.".format(package)) brew_install(package) else: log.info("Xcode and homebrew installation disabled. Proceeding on the rash assumption that packaged dependencies are in place.") log.info("Please use the --show-dependencies for more information.") # Unconditionally require openblas ("brew" is required above) if args.with_blas is None and mode == "install": config["options"]["with_blas"] = check_output(["brew", "--prefix", "openblas"])[:-1] blas_location = config["options"]["with_blas"] if not os.path.exists(blas_location): raise InstallError("Automatic determination of BLAS looked for openblas in %s but didn't find it\n" "Please provide location of a BLAS library with --with-blas (or use --with-blas=download)" % blas_location) elif osname == "Linux": # Check for apt. try: if not options["package_manager"]: raise InstallError check_call(["apt-get", "--version"]) missing_packages = [p for p in required_packages if not apt_check(p)] if missing_packages: apt_install(missing_packages) except (subprocess.CalledProcessError, InstallError): log.info("apt-get not found or disabled. Proceeding on the rash assumption that your compiled dependencies are in place.") log.info("Please use the --show-dependencies for more information.") else: blas_location = config["options"].get("with_blas") if mode == "install": if os.path.exists(firedrake_env): log.warning("Specified venv '%s' already exists", firedrake_env) quit("Can't install into existing venv '%s'" % firedrake_env) log.info("Creating firedrake venv in '%s'." % firedrake_env) # Debian's Python3 is screwed, they don't ship ensurepip as part # of the base python package, so the default python -m venv # doesn't work. Moreover, they have spiked the file such that it # calls sys.exit, which will kill any attempts to create a venv # with pip. try: import ensurepip # noqa: F401 with_pip = True except ImportError: with_pip = False import venv venv.EnvBuilder(with_pip=with_pip).create(firedrake_env) if not with_pip: import urllib.request log.debug("ensurepip unavailable, bootstrapping pip using get-pip.py") urllib.request.urlretrieve("https://bootstrap.pypa.io/get-pip.py", filename="get-pip.py") check_call([python, "get-pip.py"]) log.debug("bootstrapping pip succeeded") log.debug("Removing get-pip.py") os.remove("get-pip.py") # We haven't activated the venv so we need to manually set the environment. os.environ["VIRTUAL_ENV"] = firedrake_env # Ensure pip, setuptools, hatchling and wheel are at the latest version. # numpy requires setuptools==59.2.0 and wheel==0.37.0 until it moves to meson build run_pip(["install", "-U", "setuptools==59.2.0"]) run_pip(["install", "-U", "hatch"]) run_pip(["install", "-U", "editables"]) run_pip(["install", "-U", "pip"]) run_pip(["install", "-U", "wheel==0.37.0"]) # Pin Cython because it's causing multiple packages to fail to build run_pip(["install", "-U", "Cython==0.29.36"]) # Loopy has additional build dependencies run_pip(["install", "-U", "scikit-build"]) run_pip(["install", "-U", "nanobind"]) run_pip(["install", "-U", "make"]) run_pip(["install", "-U", "pcpp"]) petsc_dir, petsc_arch = get_petsc_dir() if options["slepc"]: slepc_dir, _ = get_slepc_dir() if mode == "install": should_link = not args.honour_petsc_dir or options.get("mpicc") is not None # Set up the MPI wrappers for opt in ["mpicc", "mpicxx", "mpif90", "mpiexec"]: if options.get(opt): name = options[opt] src = shutil.which(name) else: src = os.path.join(petsc_dir, petsc_arch, "bin", opt) dest = os.path.join(firedrake_env, "bin", opt) # Remove old symlinks if os.path.lexists(dest): os.remove(dest) elif os.path.exists(dest): os.remove(dest) if should_link: log.debug("Creating a wrapper for %s from %s to %s" % (opt, src, dest)) # use low-level os.open to ensure permissions (incl. exec.) are set according to umask fd = os.open(dest, os.O_CREAT | os.O_WRONLY) with os.fdopen(fd, "w") as f: f.write('#!/bin/bash' + os.linesep + src + ' "$@"') cc = options["mpicc"] cxx = options["mpicxx"] f90 = options["mpif90"] mpiexec = options["mpiexec"] compiler_env = create_compiler_env(cc, cxx, f90) os.chdir(firedrake_env) # Dict to store BLAS and OPENBLAS environment variables blas = dict() if blas_location == "download": blas_location = os.path.join(petsc_dir, petsc_arch) if blas_location is not None: blas["BLAS"] = blas_location blas["OPENBLAS"] = blas_location if osname == "Darwin": blas["LDFLAGS"] = "-L" + brew_gcc_libdir() log.info("BLAS config is %s" % blas) # Configuration is now complete, and we have a venv so we dump the # config file in order to maximise the chance that firedrake-update # can be run on a partial install. config_location = os.path.join(firedrake_env, ".configuration.json") json_output = json.dumps(config) with open(config_location, "w") as f: f.write(json_output) log.info("Configuration saved to " + config_location) def is_installed(name): # Return if package `name` is installed. try: importlib.import_module(name) except ModuleNotFoundError: return False else: return True if mode == "install": os.mkdir("src") os.chdir("src") if os.environ.get("FIREDRAKE_CI_TESTS", ""): # When building Firedrake in CI, don't clone Firedrake again. check_call(["ln", "-s", "../../firedrake", "firedrake"]) else: git_clone("git+https://github.com/firedrakeproject/firedrake.git") # Build the update script as soon as possible so that installation # recovery can be attempted if required. build_update_script() # Force Cython to install first to work around pip dependency issues. run_pip_install(["Cython>=0.22"]) # Pre-install requested packages if args.pip_packages is not None: for package in args.pip_packages: log.info("Pip installing %s to venv" % package) run_pip_install_wrap(package.split(), {}) # numpy requires old setuptools (+wheel) log.info("Installing numpy using setuptools==59.2.0 and wheel==0.37.0 and Cython==0.29.36") log.info("Installing numpy==1.24 due to performance regression") # https://github.com/inducer/pytential/issues/211 run_pip_install(["numpy==1.24"]) log.info("Updating setuptools and wheel to latest versions") run_pip(["install", "-U", "setuptools"]) run_pip(["install", "-U", "wheel"]) # Install VTK if not args.no_vtk: log.info("Pip installing VTK to venv") run_pip_install_wrap(["vtk>=9.0.1"], {}) if sys.version_info[:2] >= (3, 9): # Also lazy-object-proxy run_pip(["install", "lazy-object-proxy==1.4.*"]) packages = clone_dependencies("firedrake") packages = clone_dependencies("PyOP2") + ["petsc4py"] + packages packages += ["firedrake"] for p in options["packages"]: name = git_clone(p) packages.extend(clone_dependencies(name)) packages += [name] if args.honour_petsc_dir: packages.remove("petsc") # Need to install petsc first in order to resolve hdf5 dependency. if not args.honour_petsc_dir: with environment(**compiler_env): with pipargs("--no-deps"): packages.remove("petsc") install("petsc/") os.environ["PETSC_DIR"] = petsc_dir os.environ["PETSC_ARCH"] = petsc_arch # If petsc built mpich then we now need to set the compiler environment variables. if not compiler_env: compilerbin = os.path.join(petsc_dir, petsc_arch, "bin") cc = os.path.join(compilerbin, "mpicc") cxx = os.path.join(compilerbin, "mpicxx") f90 = os.path.join(compilerbin, "mpif90") mpiexec = os.path.join(compilerbin, "mpiexec") compiler_env = create_compiler_env(cc, cxx, f90) # Make sure that we link against the right MPI and PETSc shared libraries link_env = { "LDFLAGS": ( "-Wl,-rpath,{petsc_dir}/{petsc_arch}/lib -L{petsc_dir}/{petsc_arch}/lib" .format(petsc_dir=petsc_dir, petsc_arch=petsc_arch) ), } # Install mpi4py with environment(**compiler_env, **link_env): run_pip_install(["--no-cache-dir", "mpi4py"]) for p in packages: pip_requirements(p, compiler_env) # FIAT requires recursivenodes, but does not include a requirements.txt run_pip_install(["recursivenodes"]) # Pyadjoint requires checkpoint-schedules, but does not include a requirements.txt run_pip_install(["checkpoint_schedules"]) with environment(**compiler_env): build_and_install_h5py() if "loopy" in packages: # We do want to install loopy's dependencies. install("loopy/") packages.remove("loopy") with pipargs("--no-deps"): try: packages.remove("PyOP2") packages.remove("firedrake") except ValueError: pass for p in packages: if p in parallel_packages: with environment(**compiler_env, **link_env): install(p+"/") else: install(p+"/") sys.path.append(os.getcwd() + "/" + p) with environment(**compiler_env): build_and_install_libsupermesh(cc, cxx, f90, mpiexec) with pipargs("--no-deps"), environment(**compiler_env, **link_env): for p in ("PyOP2", "firedrake"): install(p+"/") # Work around easy-install.pth bug. try: packages.remove("petsc") except ValueError: pass packages.remove("petsc4py") else: # Update mode os.chdir("src") if args.update_script: # Pull firedrake, rebuild update script, launch new script git_update("firedrake") build_update_script() os.execv(sys.executable, [sys.executable, "../bin/firedrake-update", "--no-update-script"] + sys.argv[1:]) # Pre-install requested packages if args.pip_packages is not None: for package in args.pip_packages: log.info("Pip installing %s to venv" % package) run_pip_install(package.split()) deps = OrderedDict() deps.update(list_cloned_dependencies("PyOP2")) deps.update(list_cloned_dependencies("firedrake")) for p in options["packages"]: name = split_requirements_url(p)[0] deps.update(list_cloned_dependencies(name)) deps[name] = p packages = ["petsc4py"] + list(deps.keys()) packages += ["firedrake"] # update packages. if not args.honour_petsc_dir: update_petsc = git_update("petsc", deps["petsc"]) else: update_petsc = False if os.path.exists(os.path.join(firedrake_env, "src/petsc4py")): clean("petsc4py/") shutil.move("petsc4py", "petsc4py_old") update_petsc4py = True else: # Even if petsc has been installed, petsc4py might not, in # which case we need to make sure that we install it. # This situation occurs, e.g., when `firedrake-install` # fails somewhere after successfully installing petsc, # and the user continues installation with `firedrake-update`. update_petsc4py = not is_installed("petsc4py") if os.path.exists(os.path.join(firedrake_env, "src/slepc4py")): clean("slepc4py") shutil.move("slepc4py", "slepc4py_old") packages.remove("petsc") packages.remove("petsc4py") if args.clean: clean_obsolete_packages() for package in packages: pip_uninstall(package) if args.rebuild: pip_uninstall("petsc4py") # If there is a petsc package to remove, then we're an old installation which will need to rebuild PETSc. update_petsc = pip_uninstall("petsc") or update_petsc for p in packages: try: git_update(p, deps.get(p, None)) except OSError as e: if e.errno == 2: log.warning("%s missing, cloning anew.\n" % p) git_clone(deps[p]) else: raise with pipargs("--no-deps"): with environment(**compiler_env): # Only rebuild petsc if it has changed. if not args.honour_petsc_dir and (args.rebuild or update_petsc): clean("petsc/") install("petsc/") if not args.honour_petsc_dir: os.environ["PETSC_DIR"] = petsc_dir os.environ["PETSC_ARCH"] = petsc_arch # If petsc built mpich then we now need to set the compiler environment variables. if not compiler_env: compilerbin = os.path.join(petsc_dir, petsc_arch, "bin") cc = os.path.join(compilerbin, "mpicc") cxx = os.path.join(compilerbin, "mpicxx") f90 = os.path.join(compilerbin, "mpif90") mpiexec = os.path.join(compilerbin, "mpiexec") compiler_env = create_compiler_env(cc, cxx, f90) # Make sure that we link against the right MPI and PETSc shared libraries link_env = { "LDFLAGS": ( "-Wl,-rpath,{petsc_dir}/{petsc_arch}/lib -L{petsc_dir}/{petsc_arch}/lib" .format(petsc_dir=petsc_dir, petsc_arch=petsc_arch) ), } # Install mpi4py with environment(**compiler_env, **link_env): run_pip_install(["--no-cache-dir", "mpi4py"]) # update dependencies. for p in packages: pip_requirements(p, compiler_env) # FIAT requires recursivenodes, but does not include a requirements.txt run_pip_install(["recursivenodes"]) # Pyadjoint requires checkpoint-schedules, but does not include a requirements.txt run_pip_install(["checkpoint_schedules"]) with pipargs("--no-deps"): if args.rebuild or update_petsc or update_petsc4py: # If not honour_petsc_dir, we have a version of petsc that contains # petsc4py in its source tree (old petsc must have been updated # in the above). # If honour_petsc_dir, we can get here (e.g., with --rebuild flag) # without obtaining the latest petsc in the above: # thus, we must check if the user installed petsc contains petsc4py. if args.honour_petsc_dir: if not os.path.exists(get_petsc4py_dir()): raise InstallError("""\npetsc4py not found in your installation of PETSc, which indicates that your PETSc is relatively old. Please consider updating your PETSc manually. """) clean(get_petsc4py_dir()) with environment(**compiler_env): install("petsc4py/") # Always rebuild h5py. with environment(**compiler_env): build_and_install_h5py() if "loopy" in packages: # We do want to install loopy's dependencies. clean("loopy") install("loopy/") packages.remove("loopy") try: log.info("Attempting to install symengine") run_pip_install(["symengine"]) except subprocess.CalledProcessError: log.info("Symengine installation failed, this is NOT FATAL.") with pipargs("--no-deps"): try: packages.remove("PyOP2") packages.remove("firedrake") except ValueError: pass for p in packages: clean(p) if p in parallel_packages: with environment(**compiler_env, **link_env): install(p+"/") else: install(p+"/") with environment(**compiler_env): build_and_install_libsupermesh(cc, cxx, f90, mpiexec) with pipargs("--no-deps"), environment(**compiler_env, **link_env): for p in ("PyOP2", "firedrake"): install(p+"/") # Ensure pytest is at the latest version run_pip(["install", "-U", "pytest"]) with environment(**compiler_env): with pipargs("--no-deps"): if options["slepc"]: build_and_install_slepc() with environment(PETSC_DIR=petsc_dir, SLEPC_DIR=slepc_dir, PETSC_ARCH=petsc_arch): install("slepc4py/") if options["tinyasm"]: build_and_install_tinyasm() if options["netgen"]: build_and_install_netgen() with pipargs("--no-deps"): if options["opencascade"]: build_and_install_pythonocc() with pipargs("--no-deps"): if options["torch"]: build_and_install_torch() if args.documentation_dependencies: install_documentation_dependencies() if mode == "update": try: import firedrake_configuration firedrake_configuration.setup_cache_dirs() log.info("Clearing just in time compilation caches.") from firedrake.tsfc_interface import clear_cache, TSFCKernel from pyop2.compilation import clear_cache as pyop2_clear_cache print('Removing cached TSFC kernels from %s' % TSFCKernel._cachedir) clear_cache() pyop2_clear_cache() except: # noqa: E722 # Unconditional except in order to avoid upgrade script failures. log.error("Failed to clear caches. Try running firedrake-clean.") os.chdir("../..") if mode == "install": log.info("\n\nSuccessfully installed Firedrake.\n") log.info("\nFiredrake has been installed in a python venv. You activate it with:\n") log.info(" . %s/bin/activate\n" % firedrake_env) log.info("The venv can be deactivated by running:\n") log.info(" deactivate\n\n") log.info("To upgrade Firedrake activate the venv and run firedrake-update\n") else: log.info("\n\nSuccessfully updated Firedrake.\n")