#! /usr/bin/env python3
import logging
import platform
import subprocess
import sys
import os
import stat
import shutil
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import argparse
from collections import OrderedDict
import atexit

# Packages which we wish to ensure are always recompiled
wheel_blacklist = ["mpi4py", "randomgen", "islpy"]

# Firedrake application installation shortcuts.
firedrake_apps = {
    "gusto": ("""Atmospheric dynamical core library. http://firedrakeproject.org/gusto""",
              "git+ssh://github.com/firedrakeproject/gusto#egg=gusto"),
    "thetis": ("""Coastal ocean model. http://thetisproject.org""",
               "git+ssh://github.com/thetisproject/thetis#egg=thetis"),
    "pyadjoint": ("""New generation adjoint. https://bitbucket.org/dolfin-adjoint/pyadjoint""",
                  "git+ssh://bitbucket.org/dolfin-adjoint/pyadjoint.git#egg=pyadjoint"),
    "icepack": ("""Glacier and ice sheet model. https://icepack.github.io""",
                "git+ssh://github.com/icepack/icepack.git#egg=icepack"),
}


class InstallError(Exception):
    # Exception for generic install problems.
    pass


class FiredrakeConfiguration(dict):
    """A dictionary extended to facilitate the storage of Firedrake
    configuration information."""
    def __init__(self, args=None):
        super(FiredrakeConfiguration, self).__init__()

        '''A record of the persistent options in force.'''
        self["options"] = {}
        '''Relevant environment variables.'''
        self["environment"] = {}
        '''Additional packages installed via the plugin interface.'''
        self["additions"] = []

        if args:
            for o in self._persistent_options:
                if o in args.__dict__.keys():
                    self["options"][o] = args.__dict__[o]

    _persistent_options = ["package_manager",
                           "minimal_petsc", "mpicc", "mpicxx", "mpif90", "mpiexec", "disable_ssh",
                           "honour_petsc_dir", "with_parmetis",
                           "slepc", "packages", "honour_pythonpath",
                           "opencascade",
                           "petsc_int_type", "cache_dir"]


def deep_update(this, that):
    from collections import abc
    for k, v in that.items():
        if isinstance(v, abc.Mapping) and k in this.keys():
            this[k] = deep_update(this.get(k, {}), v)
        else:
            this[k] = v
    return this


invocation_directory = os.path.abspath(os.getcwd())

if os.path.basename(__file__) == "firedrake-install":
    mode = "install"
elif os.path.basename(__file__) == "firedrake-update":
    mode = "update"
    os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../..")
else:
    sys.exit("Script must be invoked either as firedrake-install or firedrake-update")


# Set up logging
# Log to file at DEBUG level
logfile = os.path.join(invocation_directory, 'firedrake-%s.log' % mode)
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(levelname)-6s %(message)s',
                    filename=logfile,
                    filemode='w')
# Log to console at INFO level
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logging.getLogger().addHandler(console)

log = logging.getLogger()

log.info("Running %s" % " ".join(sys.argv))


if sys.version_info < (3, 5):
    if mode == "install":
        print("""\nInstalling Firedrake requires Python 3, at least version 3.5.
You should run firedrake-install with python3.""")
    if mode == "update":
        if hasattr(sys, "real_prefix"):
            # sys.real_prefix exists iff we are in an active virtualenv.
            #
            # Existing install trying to update past the py2/py3 barrier
            print("""\nFiredrake is now Python 3 only.  You cannot upgrade your existing installation.
Please follow the instructions at http://www.firedrakeproject.org/download.html to reinstall.""")
            sys.exit(1)
        else:
            # Accidentally (?) running firedrake-update with python2.
            print("""\nfiredrake-update must be run with Python 3, did you accidentally use Python 2?""")
    sys.exit(1)


branches = {}

jenkins = "JENKINS_URL" in os.environ
ci_testing_firedrake = "FIREDRAKE_CI_TESTS" in os.environ


def resolve_doi_branches(doi):
    import requests
    import hashlib
    log.info("Installing Firedrake components specified by {}".format(doi))
    response = requests.get("https://zenodo.org/api/records",
                            params={"q": "doi:{}".format(doi.replace("/", r"\/"))})
    if response.status_code >= 400:
        log.error("Unable to obtain Zenodo record for doi {}".format(doi))
        log.error("Response was {}".format(response.json()))
        sys.exit(1)
    response = response.json()
    try:
        record, = response["hits"]["hits"]
    except ValueError:
        log.error("Was expecting one record for doi '{doi}', found {num}".format(
            doi=doi, num=response["hits"]["total"]))
        log.error("Response was {}".format(response))
        sys.exit(1)
    files = record["files"]
    try:
        componentjson, = (f for f in files if f["key"] == "components.json")
    except ValueError:
        log.error("Expecting to find exactly one 'components.json' in record")
        sys.exit(1)
    download = requests.get(componentjson["links"]["self"])
    if download.status_code >= 400:
        log.error("Unable to download 'components.json'")
        log.error("Response was {}".format(download.json()))
        sys.exit(1)
    # component response has checksum as "md5:HEXDIGEST", strip the md5.
    if hashlib.md5(download.content).hexdigest() != componentjson["checksum"][4:]:
        log.error("Download failed checksum, expecting {expect}, got {got}".format(
            expect=componentjson["checksum"][4:],
            got=hashlib.md5(download.content).hexdigest()))
        sys.exit(1)
    componentjson = download.json()
    branches = {}
    for record in componentjson["components"]:
        commit = record["commit"]
        component = record["component"]
        package = component[component.find("/")+1:].lower()
        branches[package] = commit
        log.info("Using commit {commit} for component {comp}".format(
            commit=commit, comp=component))
    return branches


if mode == "install":
    # Handle command line arguments.
    parser = ArgumentParser(description="""Install firedrake and its dependencies.""",
                            epilog="""The install process has three steps.

1. Any required system packages are installed using brew (MacOS) or apt (Ubuntu
   and similar Linux systems). On a Linux system without apt, the installation
   will fail if a dependency is not found.

2. A set of standard and/or third party Python packages is installed to the
   specified install location.

3. The core set of Python packages is downloaded to ./firedrake/src/ and
   installed to the specified location.

The install creates a venv in ./firedrake and installs inside
that venv.

The installer will ensure that the required configuration options are
passed to PETSc. In addition, any configure options which you provide
in the PETSC_CONFIGURE_OPTIONS environment variable will be
honoured.""",
                            formatter_class=RawDescriptionHelpFormatter)

    parser.add_argument("--slepc", action="store_true",
                        help="Install SLEPc along with PETSc.")
    parser.add_argument("--opencascade", action="store_true",
                        help="Install OpenCASCADE for CAD integration.")
    parser.add_argument("--disable-ssh", action="store_true",
                        help="Do not attempt to use ssh to clone git repositories: fall immediately back to https.")
    parser.add_argument("--no-package-manager", action='store_false', dest="package_manager",
                        help="Do not attempt to use apt or homebrew to install operating system packages on which we depend.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--minimal-petsc", action="store_true",
                       help="Minimise the set of petsc dependencies installed. This creates faster build times (useful for build testing).")
    group.add_argument("--with-parmetis", action="store_true",
                       help="Install PETSc with ParMETIS? (Non-free license, see http://glaros.dtc.umn.edu/gkhome/metis/parmetis/download)")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--honour-petsc-dir", action="store_true",
                       help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.")
    group.add_argument("--petsc-int-type", choices=["int32", "int64"],
                       default="int32", type=str,
                       help="The integer type used by PETSc.  Use int64 if you need to solve problems with more than 2 billion degrees of freedom.  Only takes effect if firedrake-install builds PETSc.")

    parser.add_argument("--honour-pythonpath", action="store_true",
                        help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
    parser.add_argument("--rebuild-script", action="store_true",
                        help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
    parser.add_argument("--doi", type=str, nargs=1,
                        help="Install a set of components matching a particular Zenodo DOI. The record should have been created with firedrake-zenodo.")
    # Used for testing if Zenodo broke the API
    # Tries to resolve to a known DOI and immediately exits.
    parser.add_argument("--test-doi-resolution", action="store_true",
                        help=argparse.SUPPRESS)
    parser.add_argument("--package-branch", type=str, nargs=2, action="append", metavar=("PACKAGE", "BRANCH"),
                        help="Specify which branch of a package to use. This takes two arguments, the package name and the branch.")
    parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
    parser.add_argument("--mpicc", type=str,
                        action="store", default=None,
                        help="C compiler to use when building with MPI (default is 'mpicc')")
    parser.add_argument("--mpicxx", type=str,
                        action="store", default=None,
                        help="C++ compiler to use when building with MPI (default is 'mpicxx')")
    parser.add_argument("--mpif90", type=str,
                        action="store", default=None,
                        help="Fortran compiler to use when building with MPI (default is 'mpif90')")
    parser.add_argument("--mpiexec", type=str,
                        action="store", default=None,
                        help="MPI launcher (default is 'mpiexec')")
    parser.add_argument("--show-petsc-configure-options", action="store_true",
                        help="Print out the configure options passed to PETSc and exit")
    parser.add_argument("--venv-name", default="firedrake",
                        type=str, action="store",
                        help="Name of the venv to create (default is 'firedrake')")
    parser.add_argument("--install", action="append", dest="packages",
                        help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir . Some additional packages have shortcut install options for more information see --install-help.")
    parser.add_argument("--install-help", action="store_true",
                        help="Provide information on packages which can be installed using shortcut names.")
    parser.add_argument("--cache-dir", type=str,
                        action="store",
                        help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).")
    parser.add_argument("--documentation-dependencies", action="store_true",
                        help="Install the dependencies required to build the documentation")

    args = parser.parse_args()

    # If the user has set any MPI info, they must set them all
    if args.mpicc or args.mpicxx or args.mpif90 or args.mpiexec:
        if not (args.mpicc and args.mpicxx and args.mpif90 and args.mpiexec):
            log.error("If you set any MPI information, you must set all of {mpicc, mpicxx, mpif90, mpiexec}.")
            sys.exit(1)

    if args.package_branch:
        branches = {package.lower(): branch for package, branch in args.package_branch}
    if args.test_doi_resolution:
        actual = resolve_doi_branches("10.5281/zenodo.1322546")
        expect = {'coffee': '87e50785d3a05b111f5423a66d461cd44cc4bdc9',
                  'finat': 'aa74fd499304c8363a4520555fd62ef21e8e5e1f',
                  'fiat': '184601a46c24fb5cbf8fd7961d22b16dd26890e7',
                  'firedrake': '6a30b64da01eb587dcc0e04e8e6b84fe4839bdb7',
                  'petsc': '413f72f04f5cb0a010c85e03ed029573ff6d4c63',
                  'petsc4py': 'ac2690070a80211dfdbab04634bdb3496c14ca0a',
                  'tsfc': 'fe9973eaacaa205fd491cd1cc9b3743b93a3d076',
                  'ufl': 'c5eb7fbe89c1091132479c081e6fa9c182191dcc',
                  'pyop2': '741a21ba9a62cb67c0aa300a2e199436ea8cb61c'}
        if actual != expect:
            log.error("Unable to resolve DOI correctly.")
            log.error("You'll need to figure out how Zenodo have changed their API.")
            sys.exit(1)
        else:
            log.info("DOI resolution test passed.")
            sys.exit(0)
    if args.doi:
        branches = resolve_doi_branches(args.doi[0])

    args.prefix = False  # Disabled as untested
    args.packages = args.packages or []

    config = FiredrakeConfiguration(args)
else:
    import firedrake_configuration
    config = firedrake_configuration.get_config()
    if config is None:
        raise InstallError("Failed to find existing Firedrake configuration")

    parser = ArgumentParser(description="""Update this firedrake install to the latest versions of all packages.""",
                            formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument("--no-update-script", action="store_false", dest="update_script",
                        help="Do not update script before updating Firedrake.")
    parser.add_argument("--rebuild", action="store_true",
                        help="Rebuild all packages even if no new version is available. Usually petsc and petsc4py are only rebuilt if they change. All other packages are always rebuilt.")
    parser.add_argument("--rebuild-script", action="store_true",
                        help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
    parser.add_argument("--slepc", action="store_true", dest="slepc", default=config["options"]["slepc"],
                        help="Install SLEPc along with PETSc")
    parser.add_argument("--opencascade", action="store_true", dest="opencascade", default=config["options"].get("opencascade", False),
                        help="Install OpenCASCADE for CAD integration.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--honour-petsc-dir", action="store_true",
                       default=config["options"].get("honour_petsc_dir", False),
                       help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.")
    group.add_argument("--petsc-int-type", choices=["int32", "int64"],
                       default="int32", type=str,
                       help="The integer type used by PETSc.  Use int64 if you need to solve problems with more than 2 billion degrees of freedom.  Only takes effect if firedrake-install builds PETSc.")

    parser.add_argument("--honour-pythonpath", action="store_true", default=config["options"].get("honour_pythonpath", False),
                        help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
    parser.add_argument("--clean", action='store_true',
                        help="Delete any remnants of obsolete Firedrake components.")
    parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
    parser.add_argument("--install", action="append", dest="packages", default=config["options"].get("packages", []),
                        help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir. Some additional packages have shortcut install options for more information see --install-help.")
    parser.add_argument("--install-help", action="store_true",
                        help="Provide information on packages which can be installed using shortcut names.")
    parser.add_argument("--cache-dir", type=str,
                        action="store", default=config["options"].get("cache_dir", ""),
                        help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).")
    parser.add_argument("--documentation-dependencies", action="store_true",
                        help="Install the dependencies required to build the documentation")

    args = parser.parse_args()

    args.packages = list(set(args.packages))  # remove duplicates

    petsc_int_type_changed = False
    if config["options"].get("petsc_int_type", "int32") != args.petsc_int_type:
        petsc_int_type_changed = True
        args.rebuild = True

    config = deep_update(config, FiredrakeConfiguration(args))

if args.install_help:
    help_string = """
You can install the following packages by passing --install shortname
where shortname is one of the names given below:

"""
    componentformat = "|{:10}|{:70}|\n"
    header = componentformat.format("Name", "Description")
    line = "-" * (len(header) - 1) + "\n"
    help_string += line + header + line
    for package, d in firedrake_apps.items():
        help_string += componentformat.format(package, d[0])
    help_string += line
    print(help_string)
    sys.exit(0)


@atexit.register
def print_log_location():
    log.info("\n\n%s log saved in %s" % (mode.capitalize(), logfile))


class directory(object):
    """Context manager that executes body in a given directory"""
    def __init__(self, dir):
        self.dir = os.path.abspath(dir)

    def __enter__(self):
        self.olddir = os.path.abspath(os.getcwd())
        log.debug("Old path '%s'" % self.olddir)
        log.debug("Pushing path '%s'" % self.dir)
        os.chdir(self.dir)

    def __exit__(self, *args):
        log.debug("Popping path '%s'" % self.dir)
        os.chdir(self.olddir)
        log.debug("New path '%s'" % self.olddir)


class environment(object):
    def __init__(self, **env):
        self.old = os.environ.copy()
        self.new = env

    def __enter__(self):
        os.environ.update(self.new)

    def __exit__(self, *args):
        os.environ = self.old


options = config["options"]

# Apply short cut package names.
options["packages"] = [firedrake_apps.get(p, (None, p))[1] for p in options["packages"]]


# Record of obsolete packages which --clean should remove from old installs.
old_git_packages = ["dolfin-adjoint", "libadjoint"]

if mode == "install":
    firedrake_env = os.path.abspath(args.venv_name)
else:
    try:
        firedrake_env = os.environ["VIRTUAL_ENV"]
    except KeyError:
        quit("Unable to retrieve venv name from the environment.\n Please ensure the venv is active before running firedrake-update.")

if "cache_dir" not in config["options"] or not config["options"]["cache_dir"]:
    config["options"]["cache_dir"] = os.path.join(firedrake_env, ".cache")


# venv install
python = ["%s/bin/python" % firedrake_env]
# Use the pip from the venv
pip = python + ["-m", "pip"]
pipinstall = pip + ["install", "--no-binary", ",".join(wheel_blacklist)]


# This context manager should be used whenever arguments should be temporarily added to pipinstall.
class pipargs(object):
    def __init__(self, *args):
        self.args = args

    def __enter__(self):
        self.save = pipinstall.copy()
        pipinstall.extend(self.args)

    def __exit__(self, *args, **kwargs):
        global pipinstall
        pipinstall = self.save


pyinstall = python + ["setup.py", "install"]

if "PYTHONPATH" in os.environ and not args.honour_pythonpath:
    quit("""The PYTHONPATH environment variable is set. This is probably an error.
If you really want to use your own Python packages, please run again with the
--honour-pythonpath option.
""")

petsc_opts = "--download-eigen=%s/src/eigen-3.3.3.tgz " % firedrake_env

petsc_opts += "--with-fortran-bindings=0 "

petsc_options = {"--with-fortran-bindings=0",
                 "--with-debugging=0",
                 "--with-shared-libraries=1",
                 "--with-c2html=0",
                 "--download-eigen=%s/src/eigen-3.3.3.tgz " % firedrake_env,
                 # File format
                 "--download-hdf5",
                 # AMG
                 "--download-hypre",
                 # Sparse direct solver
                 "--download-superlu_dist",
                 # Parallel mesh partitioner
                 "--download-ptscotch",
                 # For superlu_dist amongst others.
                 "--with-cxx-dialect=C++11"}

if not options["minimal_petsc"]:
    petsc_options.add("--with-zlib")
    # File formats
    petsc_options.add("--download-netcdf")
    petsc_options.add("--download-pnetcdf")
    petsc_options.add("--download-exodusii")
    # Sparse direct solvers
    petsc_options.add("--download-suitesparse")
    petsc_options.add("--download-pastix")
    # Serial mesh partitioner
    petsc_options.add("--download-metis")
    if options.get("with_parmetis", None):
        # Non-free license.
        petsc_options.add("--download-parmetis")

if options["petsc_int_type"] == "int32":
    # Sparse direct solver
    petsc_options.add("--download-scalapack")  # Needed for mumps
    petsc_options.add("--download-mumps")
    # Serial mesh partitioner
    petsc_options.add("--download-chaco")
    if not options["minimal_petsc"]:
        # AMG
        petsc_options.add("--download-ml")
        # Mesh adaptivity (needs metis, but not parmetis)
        petsc_options.add("--download-pragmatic")
else:
    petsc_options.add("--with-64-bit-indices")

if options["mpiexec"] is not None:
    petsc_options.add("--with-mpiexec={}".format(options["mpiexec"]))

if "PETSC_CONFIGURE_OPTIONS" not in os.environ:
    os.environ["PETSC_CONFIGURE_OPTIONS"] = " ".join(sorted(petsc_options))
else:
    all_opts = petsc_options | set(os.environ["PETSC_CONFIGURE_OPTIONS"].split())
    os.environ["PETSC_CONFIGURE_OPTIONS"] = " ".join(sorted(all_opts))


if mode == "update" and petsc_int_type_changed:
    log.warning("""Force rebuilding all packages because PETSc int type changed""")


def check_call(arguments, env=None):
    try:
        log.debug("Running command '%s'", " ".join(arguments))
        log.debug(subprocess.check_output(arguments, stderr=subprocess.STDOUT, env=env).decode())
    except subprocess.CalledProcessError as e:
        log.debug(e.output.decode())
        raise


def check_output(args, env=None):
    try:
        log.debug("Running command '%s'", " ".join(args))
        result = subprocess.check_output(args, stderr=subprocess.STDOUT, env=env)
        if isinstance(result, str):
            # Python 2
            return result
        else:
            # Python 3
            return result.decode()
    except subprocess.CalledProcessError as e:
        log.debug(e.output.decode())
        raise


def brew_install(name, options=None):
    try:
        # Check if it's already installed
        check_call(["brew", "list", name])
    except subprocess.CalledProcessError:
        # If not found, go ahead and install
        arguments = [name]
        if options:
            arguments = options + arguments
        if args.verbose:
            arguments = ["--verbose"] + arguments
        check_call(["brew", "install"] + arguments)


def apt_check(name):
    log.info("Checking for presence of package %s..." % name)
    # Note that subprocess return codes have the opposite logical
    # meanings to those of Python variables.
    try:
        check_call(["dpkg-query", "-s", name])
        log.info("  installed.")
        return True
    except subprocess.CalledProcessError:
        log.info("  missing.")
        return False


def apt_install(names):
    log.info("Installing missing packages: %s." % ", ".join(names))
    if sys.stdin.isatty():
        subprocess.check_call(["sudo", "apt-get", "install"] + names)
    else:
        log.info("Non-interactive stdin detected; installing without prompts")
        subprocess.check_call(["sudo", "apt-get", "-y", "install"] + names)


def split_requirements_url(url):
    name = url.split(".git")[0].split("#")[0].split("/")[-1]
    spliturl = url.split("://")[1].split("#")[0].split("@")
    try:
        plain_url, branch = spliturl
    except ValueError:
        plain_url = spliturl[0]
        branch = "master"
    return name, plain_url, branch


def git_url(plain_url, protocol):
    if protocol == "ssh":
        return "git@%s:%s" % tuple(plain_url.split("/", 1))
    elif protocol == "https":
        return "https://%s" % plain_url
    else:
        raise ValueError("Unknown git protocol: %s" % protocol)


def git_clone(url):
    name, plain_url, branch = split_requirements_url(url)
    if name == "petsc" and args.honour_petsc_dir:
        log.info("Using existing petsc installation\n")
        return name
    log.info("Cloning %s\n" % name)
    branch = branches.get(name.lower(), branch)
    try:
        if options["disable_ssh"]:
            raise InstallError("Skipping ssh clone because --disable-ssh")
        # note: so far only loopy requires submodule
        check_call(["git", "clone", "-q", "--recursive", git_url(plain_url, "ssh")])
        log.info("Successfully cloned repository %s" % name)
    except (subprocess.CalledProcessError, InstallError):
        if not options["disable_ssh"]:
            log.warning("Failed to clone %s using ssh, falling back to https." % name)
        try:
            check_call(["git", "clone", "-q", "--recursive", git_url(plain_url, "https")])
            log.info("Successfully cloned repository %s." % name)
        except subprocess.CalledProcessError:
            log.error("Failed to clone %s branch %s." % (name, branch))
            raise
    with directory(name):
        try:
            log.info("Checking out branch %s" % branch)
            check_call(["git", "checkout", "-q", branch])
            log.info("Successfully checked out branch %s" % branch)
        except subprocess.CalledProcessError:
            log.error("Failed to check out branch %s" % branch)
            raise
    return name


def list_cloned_dependencies(name):
    log.info("Finding dependencies of %s\n" % name)
    deps = OrderedDict()
    try:
        for dep in open(name + "/requirements-git.txt", "r"):
            name = split_requirements_url(dep.strip())[0]
            deps[name] = dep.strip()
    except IOError:
        pass
    return deps


def clone_dependencies(name):
    log.info("Cloning the dependencies of %s" % name)
    deps = []
    try:
        for dep in open(name + "/requirements-git.txt", "r"):
            deps.append(git_clone(dep.strip()))
    except IOError:
        pass
    return deps


def git_update(name, url=None):
    # Update the named git repo and return true if the current branch actually changed.
    log.info("Updating the git repository for %s" % name)
    with directory(name):
        git_sha = check_output(["git", "rev-parse", "HEAD"])
        # Ensure remotes get updated if and when we move repositories.
        if url:
            plain_url = split_requirements_url(url)[1]
            current_url = check_output(["git", "remote", "-v"]).split()[1]
            protocol = "https" if current_url.startswith("https") else "ssh"
            new_url = git_url(plain_url, protocol)
            # Ensure we only change from bitbucket to github and not the reverse.
            if new_url != current_url and "bitbucket.org" in current_url \
               and "github.com/firedrakeproject" in plain_url:
                log.info("Updating git remote for %s" % name)
                check_call(["git", "remote", "set-url", "origin", new_url])
        check_call(["git", "pull"])
        git_sha_new = check_output(["git", "rev-parse", "HEAD"])
    return git_sha != git_sha_new


def run_pip(args):
    check_call(pip + args)


def run_pip_install(pipargs):
    # Make pip verbose when logging, so we see what the
    # subprocesses wrote out.
    # Particularly important for debugging petsc fails.
    pipargs = ["-vvv"] + pipargs
    check_call(pipinstall + pipargs)


def run_cmd(args):
    check_call(args)


def pip_requirements(package):
    log.info("Installing pip dependencies for %s" % package)
    if os.path.isfile("%s/requirements-ext.txt" % package):
        run_pip_install(["-r", "%s/requirements-ext.txt" % package])
    elif os.path.isfile("%s/requirements.txt" % package):
        if package == "COFFEE":
            # FIXME: Horrible hack to work around
            # https://github.com/coin-or/pulp/issues/123
            run_pip_install(["--no-deps", "-r", "%s/requirements.txt" % package])
        elif package == "pyadjoint":
            with open("%s/requirements.txt" % package, "r") as f:
                reqs = f.readlines()
            for req in reqs:
                req = req.strip()
                try:
                    run_pip_install([req])
                except subprocess.CalledProcessError as e:
                    if req == "tensorflow":
                        log.debug("Skipping failing install of optional pyadjoint dependency tensorflow")
                    else:
                        raise e
        elif package == "loopy":
            # loopy dependencies are installed in setup.py
            return
        else:
            run_pip_install(["-r", "%s/requirements.txt" % package])
    else:
        log.info("No dependencies found. Skipping.")


def install(package):
    log.info("Installing %s" % package)
    if package == "petsc/":
        build_and_install_petsc()
    # The following outrageous hack works around the fact that petsc and co. cannot be installed in developer mode.
    elif package not in ["petsc4py/", "slepc/", "slepc4py/"]:
        run_pip_install(["-e", package])
    else:
        run_pip_install(["--ignore-installed", package])


def clean(package):
    log.info("Cleaning %s" % package)
    with directory(package):
        check_call(["python", "setup.py", "clean"])


def pip_uninstall(package):
    log.info("Removing existing %s installations\n" % package)
    # Uninstalling something with pip is an absolute disaster.  We
    # have to use pip freeze to list all available packages "locally"
    # and keep on removing the one we want until it is gone from this
    # list!  Yes, pip will happily have two different versions of the
    # same package co-existing.  Moreover, depending on the phase of
    # the moon, the order in which they are uninstalled is not the
    # same as the order in which they appear on sys.path!
    again = True
    i = 0
    while again:
        # List installed packages, "locally".  In a venv,
        # this just tells me packages in the venv, otherwise it
        # gives me everything.
        lines = check_output(pip + ["freeze", "-l"])
        again = False
        for line in lines.split("\n"):
            # Do we have a locally installed package?
            if line.startswith(package):
                # Uninstall it.
                run_pip(["uninstall", "-y", line.strip()])
                # Go round again, because THERE MIGHT BE ANOTHER ONE!
                again = True
        i += 1
        if i > 10:
            raise InstallError("pip claims it uninstalled %s more than 10 times.  Something is probably broken.", package)


def get_petsc_dir():
    if args.honour_petsc_dir:
        try:
            petsc_dir = os.environ["PETSC_DIR"]
        except KeyError:
            raise InstallError("Unable to find installed PETSc (did you forget to set PETSC_DIR?)")
        petsc_arch = os.environ.get("PETSC_ARCH", "")
    else:
        try:
            petsc_dir = check_output(python
                                     + ["-c", "import petsc; print(petsc.get_petsc_dir())"]).strip()

            petsc_arch = ""
        except subprocess.CalledProcessError:
            raise InstallError("Unable to find installed PETSc")
    return petsc_dir, petsc_arch


def get_slepc_dir():
    petsc_dir, petsc_arch = get_petsc_dir()
    if args.honour_petsc_dir:
        try:
            slepc_dir = os.environ["SLEPC_DIR"]
        except KeyError:
            raise InstallError("Need to set SLEPC_DIR for --slepc with --honour-petsc-dir")
    else:
        try:
            slepc_dir = check_output(python
                                     + ["-c", "import slepc; print(slepc.get_slepc_dir())"]).strip()
        except subprocess.CalledProcessError:
            raise InstallError("Unable to find installed SLEPc")
    return slepc_dir, petsc_arch


def build_and_install_petsc():
    import hashlib
    import urllib.request
    tarball = "eigen-3.3.3.tgz"
    url = "https://bitbucket.org/eigen/eigen/get/3.3.3.tar.gz"
    sha = hashlib.sha256()
    expect = "94878cbfa27b0d0fbc64c00d4aafa137f678d5315ae62ba4aecddbd4269ae75f"
    if not os.path.exists(tarball):
        log.info("Downloading Eigen from '%s' to '%s'" % (url, tarball))
        urllib.request.urlretrieve(url, filename=tarball)
    else:
        log.info("Eigen tarball already downloaded to '%s'" % tarball)
    log.info("Checking Eigen tarball integrity")
    with open(tarball, "rb") as f:
        while True:
            data = f.read(65536)
            if not data:
                break
            sha.update(data)
    actual = sha.hexdigest()
    if actual != expect:
        raise InstallError("Downloaded Eigen tarball has incorrect sha256sum, expected '%s', was '%s'",
                           expect, actual)
    else:
        log.info("Eigen tarball hash valid")
    log.info("Building PETSc. \nDepending on your platform, may take between a few minutes and an hour or more to build!")
    run_pip_install(["--ignore-installed", "petsc/"])


def build_and_install_h5py():
    import shutil
    log.info("Installing h5py")
    # Clean up old downloads
    if os.path.exists("h5py-2.5.0"):
        log.info("Removing old h5py-2.5.0 source")
        shutil.rmtree("h5py-2.5.0")
    if os.path.exists("h5py.tar.gz"):
        log.info("Removing old h5py.tar.gz")
        os.remove("h5py.tar.gz")

    url = "git+https://github.com/firedrakeproject/h5py.git@firedrake#egg=h5py"
    if os.path.exists("h5py"):
        changed = False
        with directory("h5py"):
            # Rewrite old h5py/h5py remote to firedrakeproject/h5py remote.
            plain_url = split_requirements_url(url)[1]
            current_remote = check_output(["git", "remote", "-v"]).split()[1]
            proto = "https" if current_remote.startswith("https") else "ssh"
            new_remote = git_url(plain_url, proto)
            if new_remote != current_remote and "h5py/h5py.git" in current_remote:
                log.info("Updating git remote for h5py from %s to %s", current_remote, new_remote)
                check_call(["git", "remote", "set-url", "origin", new_remote])
                check_call(["git", "fetch", "-p", "origin"])
                check_call(["git", "checkout", "firedrake"])
                changed = True
        changed |= git_update("h5py")
    else:
        git_clone(url)
        changed = True

    petsc_dir, petsc_arch = get_petsc_dir()
    hdf5_dir = "%s/%s" % (petsc_dir, petsc_arch)
    if changed or args.rebuild:
        log.info("Linking h5py against PETSc found in %s\n" % hdf5_dir)
        with environment(HDF5_DIR=hdf5_dir,
                         HDF5_MPI="ON"):
            # Only uninstall if things changed.
            pip_uninstall("h5py")
            # Pip installing from dirty directory is potentially unsafe.
            with directory("h5py/"):
                check_call(["git", "clean", "-fdx"])
            install("h5py/")
    else:
        log.info("No need to rebuild h5py")


def build_and_install_libsupermesh():
    log.info("Installing libsupermesh")
    url = "git+https://bitbucket.org/libsupermesh/libsupermesh.git"
    if os.path.exists("libsupermesh"):
        log.info("Updating the git repository for libsupermesh")
        with directory("libsupermesh"):
            check_call(["git", "fetch"])
            git_sha = check_output(["git", "rev-parse", "HEAD"])
            git_sha_new = check_output(["git", "rev-parse", "@{u}"])
            changed = git_sha != git_sha_new
            if changed:
                check_call(["git", "reset", "--hard"])
                check_call(["git", "pull"])
    else:
        git_clone(url)
        changed = True
    if changed:
        with directory("libsupermesh"):
            check_call(["git", "reset", "--hard"])
            check_call(["git", "clean", "-f", "-x", "-d"])
            check_call(["mkdir", "-p", "build"])
            with directory("build"):
                check_call(["cmake", "..", "-DBUILD_SHARED_LIBS=ON",
                            "-DCMAKE_INSTALL_PREFIX=" + firedrake_env,
                            "-DMPI_C_COMPILER=" + cc,
                            "-DMPI_CXX_COMPILER=" + cxx,
                            "-DMPI_Fortran_COMPILER=" + f90])
                check_call(["make"])
                check_call(["make", "install"])
    else:
        log.info("No need to rebuild libsupermesh")


def build_and_install_pythonocc():
    log.info("Installing pythonocc-core")
    url = "git+https://github.com/tpaviot/pythonocc-core.git@595b0a4e8e60e8d6011bea0cdb54ac878efcfcd2"
    if not os.path.exists("pythonocc-core"):
        git_clone(url)
        with directory("pythonocc-core"):
            check_call(["git", "reset", "--hard"])
            check_call(["git", "clean", "-f", "-x", "-d"])
            check_call(["mkdir", "-p", "build"])
            with directory("build"):
                check_call(["cmake", "..",
                            "-DCMAKE_INSTALL_PREFIX=" + firedrake_env,
                            "-DPYTHON_EXECUTABLE=" + python[0],
                            "-DMPI_C_COMPILER=" + cc,
                            "-DMPI_CXX_COMPILER=" + cxx,
                            "-DMPI_Fortran_COMPILER=" + f90])
                check_call(["make", "install"])
    else:
        log.info("No need to rebuild pythonocc-core")


def build_and_install_libspatialindex():
    log.info("Installing libspatialindex")
    if os.path.exists("libspatialindex"):
        log.info("Updating the git repository for libspatialindex")
        with directory("libspatialindex"):
            check_call(["git", "fetch"])
            git_sha = check_output(["git", "rev-parse", "HEAD"])
            git_sha_new = check_output(["git", "rev-parse", "@{u}"])
            libspatialindex_changed = git_sha != git_sha_new
            if libspatialindex_changed:
                check_call(["git", "reset", "--hard"])
                check_call(["git", "pull"])
    else:
        git_clone("git+https://github.com/firedrakeproject/libspatialindex.git")
        libspatialindex_changed = True

    if libspatialindex_changed:
        with directory("libspatialindex"):
            # Clean source directory
            check_call(["git", "reset", "--hard"])
            check_call(["git", "clean", "-f", "-x", "-d"])
            # Patch Makefile.am to skip building test
            check_call(["sed", "-i", "-e", "/^SUBDIRS/s/ test//", "Makefile.am"])
            # Build and install
            check_call(["./autogen.sh"])
            check_call(["./configure", "--prefix=" + firedrake_env,
                        "--enable-shared", "--disable-static"])
            check_call(["make"])
            check_call(["make", "install"])
    else:
        log.info("No need to rebuild libspatialindex")


def build_and_install_slepc():
    petsc_dir, petsc_arch = get_petsc_dir()

    if args.honour_petsc_dir:
        slepc_dir, slepc_arch = get_slepc_dir()
        log.info("Using installed SLEPc from %s/%s", slepc_dir, slepc_arch)
    else:
        log.info("Installing SLEPc.")
        url = "git+https://github.com/firedrakeproject/slepc.git@firedrake"
        if os.path.exists("slepc"):
            slepc_changed = False
            with directory("slepc"):
                plain_url = split_requirements_url(url)[1]
                current_remote = check_output(["git", "remote", "-v"]).split()[1]
                proto = "https" if current_remote.startswith("https") else "ssh"
                new_remote = git_url(plain_url, proto)
                if new_remote != current_remote:
                    log.info("Updating git remote for SLEPc from %s to %s", current_remote, new_remote)
                    check_call(["git", "remote", "set-url", "origin", new_remote])
                    check_call(["git", "fetch", "-p", "origin"])
                    check_call(["git", "checkout", "firedrake"])
                    slepc_changed = True
            slepc_changed |= git_update("slepc")
        else:
            git_clone(url)
            slepc_changed = True
        if slepc_changed:
            install("slepc/")
        else:
            log.info("No need to rebuild SLEPc")

    log.info("Installing slepc4py.")
    url = "git+https://github.com/firedrakeproject/slepc4py.git@firedrake"
    if os.path.exists("slepc4py"):
        slepc4py_changed = False
        with directory("slepc4py"):
            plain_url = split_requirements_url(url)[1]
            current_remote = check_output(["git", "remote", "-v"]).split()[1]
            proto = "https" if current_remote.startswith("https") else "ssh"
            new_remote = git_url(plain_url, proto)
            if new_remote != current_remote:
                log.info("Updating git remote for slepc4py from %s to %s", current_remote, new_remote)
                check_call(["git", "remote", "set-url", "origin", new_remote])
                check_call(["git", "fetch", "-p", "origin"])
                check_call(["git", "checkout", "firedrake"])
                slepc4py_changed = True
        slepc4py_changed |= git_update("slepc4py")
    else:
        git_clone(url)
        slepc4py_changed = True
    if slepc4py_changed:
        install("slepc4py/")
    else:
        log.info("No need to rebuild slepc4py")


def install_documentation_dependencies():
    """Just install the required dependencies. There are no provenance
    issues here so no need to record this in the configuration dict."""
    log.info("Installing documentation dependencies")
    run_pip_install(["sphinx"])
    run_pip_install(["sphinxcontrib-bibtex"])
    run_pip_install(["git+https://github.com/sphinx-contrib/youtube.git"])


def clean_obsolete_packages():
    dirs = os.listdir(".")
    for package in old_git_packages:
        pip_uninstall(package)
        if package in dirs:
            shutil.rmtree(package)


def quit(message):
    log.error(message)
    sys.exit(1)


def build_update_script():
    log.info("Creating firedrake-update script.")
    with open("firedrake/scripts/firedrake-install", "r") as f:
        update_script = f.read()

    try:
        os.mkdir("../bin")
    except OSError:
        pass
    with open("../bin/firedrake-update", "w") as f:
        f.write(update_script)
    check_call(["chmod", "a+x", "../bin/firedrake-update"])


if mode == "install" and args.show_petsc_configure_options:
    log.info("*********************************************")
    log.info("Would build PETSc with the following options:")
    log.info("*********************************************\n")
    log.info("\n".join(os.environ["PETSC_CONFIGURE_OPTIONS"].split()))
    log.info("\nEigen will be downloaded from https://bitbucket.org/eigen/eigen/get/3.3.3.tar.gz")
    sys.exit(0)


if "PETSC_DIR" in os.environ and not args.honour_petsc_dir:
    quit("""The PETSC_DIR environment variable is set. This is probably an error.
If you really want to use your own PETSc build, please run again with the
--honour-petsc-dir option.
""")

if "PETSC_DIR" not in os.environ and args.honour_petsc_dir:
    quit("""The --honour-petsc-dir is set, but PETSC_DIR environment variable is
not defined. If you have compiled PETSc manually, set PETSC_DIR
(and optionally PETSC_ARCH) variables to point to the build directory.
""")

if "SLEPC_DIR" not in os.environ and args.honour_petsc_dir and options["slepc"]:
    quit("""If you use --honour-petsc-dir, you must also build SLEPc manually
and set the SLEPC_DIR environment variable appropriately""")

if "SLEPC_DIR" in os.environ and not args.honour_petsc_dir and options["slepc"]:
    quit("""The SLEPC_DIR environment variable is set.  If you want to use
your own SLEPc version, you must also build your own PETSc and run
with --honour-petsc-dir.""")

if platform.uname()[0] == "Darwin" and args.opencascade:
    quit("""Sorry, automatically installing opencascade on OSX hasn't been
implemented yet. (It's not a supported package in brew.)
Please contact us to get this working.""")

if args.rebuild_script:
    os.chdir(os.path.dirname(os.path.realpath(__file__)) + ("/../.."))

    build_update_script()

    log.info("Successfully rebuilt firedrake-update.\n")

    log.info("To upgrade firedrake, run firedrake-update")
    sys.exit(0)


log.debug("*** Current environment (output of 'env') ***")
log.debug(check_output(["env"]))
log.debug("\n\n")

if mode == "install" or not args.update_script:
    # Check operating system.
    osname = platform.uname()[0]
    if osname == "Darwin":
        if options["package_manager"]:

            log.info("Installing command line tools...")
            try:
                check_call(["xcode-select", "--install"])
            except subprocess.CalledProcessError:
                # expected failure if already installed
                pass

            try:
                check_call(["brew", "--version"])
            except subprocess.CalledProcessError:
                quit("Homebrew not found. Please install it using the instructions at http://brew.sh and then try again.")

            log.info("Installing required packages via homebrew. You can safely ignore warnings that packages are already installed")
            # Ensure a fortran compiler is available
            brew_install("gcc")
            brew_install("openmpi")
            brew_install("python3")
            brew_install("autoconf")
            brew_install("automake")
            brew_install("cmake")
            brew_install("libtool")
            brew_install("mercurial")
            brew_install("boost")
        else:
            log.info("Xcode and homebrew installation disabled. Proceeding on the rash assumption that packaged dependencies are in place.")

    elif osname == "Linux":
        # Check for apt.
        try:
            if not options["package_manager"]:
                raise InstallError
            check_call(["apt-get", "--version"])

            apt_packages = ["build-essential",
                            "autoconf",
                            "automake",
                            "bison",  # for ptscotch
                            "flex",   # for ptscotch
                            "cmake",
                            "gfortran",
                            "git",
                            "libblas-dev",
                            "liblapack-dev",
                            "libmpich-dev",
                            "libtool",
                            "mercurial",
                            "mpich",
                            "python3-dev",
                            "python3-pip",
                            "python3-tk",
                            "python3-venv",
                            "zlib1g-dev",
                            "libboost-dev"]  # For ROL in pyadjoint

            if args.opencascade:
                apt_packages.append("liboce-ocaf-dev")
                apt_packages.append("swig")

            missing_packages = [p for p in apt_packages if not apt_check(p)]
            if missing_packages:
                apt_install(missing_packages)

        except (subprocess.CalledProcessError, InstallError):
            log.info("apt-get not found or disabled. Proceeding on the rash assumption that your compiled dependencies are in place.")
            log.info("If this is not the case, please install the following and try again:")
            log.info("* A C and C++ compiler (for example gcc/g++ or clang), GNU make")
            log.info("* A Fortran compiler (for PETSc)")
            log.info("* MPI")
            log.info("* Blas and Lapack")
            log.info("* Git, Mercurial")
            log.info("* Python version >=3.5")
            log.info("* The Python headers")
            log.info("* autoconf, automake, libtool")
            log.info("* CMake")
            log.info("* zlib")

    else:
        log.warning("You do not appear to be running Linux or MacOS. Please do not be surprised if this install fails.")


if mode == "install":
    if os.path.exists(firedrake_env):
        log.warning("Specified venv '%s' already exists", firedrake_env)
        quit("Can't install into existing venv '%s'" % firedrake_env)

    log.info("Creating firedrake venv in '%s'." % firedrake_env)
    # Debian's Python3 is screwed, they don't ship ensurepip as part
    # of the base python package, so the default python -m venv
    # doesn't work.  Moreover, they have spiked the file such that it
    # calls sys.exit, which will kill any attempts to create a venv
    # with pip.
    try:
        import ensurepip        # noqa: F401
        with_pip = True
    except ImportError:
        with_pip = False
    import venv
    venv.EnvBuilder(with_pip=with_pip).create(firedrake_env)
    if not with_pip:
        import urllib.request
        log.debug("ensurepip unavailable, bootstrapping pip using get-pip.py")
        urllib.request.urlretrieve("https://bootstrap.pypa.io/get-pip.py", filename="get-pip.py")
        check_call(python + ["get-pip.py"])
        log.debug("bootstrapping pip succeeded")
        log.debug("Removing get-pip.py")
        os.remove("get-pip.py")
    # Ensure pip and setuptools are at the latest version.
    run_pip(["install", "-U", "setuptools"])
    run_pip(["install", "-U", "pip"])

    # We haven't activated the venv so we need to manually set the environment.
    os.environ["VIRTUAL_ENV"] = firedrake_env

    # Set up the MPI wrappers
    for opt in ["mpicc", "mpicxx", "mpif90"]:
        if options.get(opt):
            name = options[opt]
            src = shutil.which(name)
            dest = os.path.join(firedrake_env, "bin", opt)
            log.debug("Creating a symlink from %s to %s" % (src, dest))
            os.symlink(src, dest)

    if options.get("mpiexec"):
        mpiexecf = os.path.join(firedrake_env, "bin", "mpiexec")
        log.debug("Creating an mpiexec wrapper for %s" % shutil.which(args.mpiexec))
        with open(mpiexecf, "w") as mpiexec:
            contents = '#!/bin/bash' + os.linesep + shutil.which(args.mpiexec) + ' "$@"'
            mpiexec.write(contents)
        os.chmod(mpiexecf, os.stat(mpiexecf).st_mode | stat.S_IEXEC)

cc = options["mpicc"] or "mpicc"
cxx = options["mpicxx"] or "mpicxx"
f90 = options["mpif90"] or "mpif90"
compiler_env = dict(MPICC=cc,
                    MPICXX=cxx,
                    MPIF90=f90,
                    MPI_C_COMPILER=cc,
                    MPI_CXX_COMPILER=cxx,
                    MPI_Fortran_COMPILER=f90,
                    CC=cc,
                    CXX=cxx,
                    F90=f90)
os.chdir(firedrake_env)

if mode == "install":
    os.mkdir("src")
    os.chdir("src")

    if jenkins and ci_testing_firedrake:
        check_call(["ln", "-s", "../../../", "firedrake"])
    else:
        git_clone("git+https://github.com/firedrakeproject/firedrake.git")

    packages = clone_dependencies("firedrake")
    packages = clone_dependencies("PyOP2") + packages
    packages += ["firedrake"]

    for p in options["packages"]:
        name = git_clone(p)
        packages.extend(clone_dependencies(name))
        packages += [name]

    if args.honour_petsc_dir:
        packages.remove("petsc")

    # Force Cython to install first to work around pip dependency issues.
    run_pip_install(["Cython>=0.22"])

    with environment(**compiler_env):
        # Need to install petsc first in order to resolve hdf5 dependency.
        if not args.honour_petsc_dir:
            with pipargs("--no-deps"):
                packages.remove("petsc")
                install("petsc/")

        for p in packages:
            pip_requirements(p)

        build_and_install_h5py()
        build_and_install_libspatialindex()
        build_and_install_libsupermesh()

        if "loopy" in packages:
            # We do want to install loopy's dependencies.
            install("loopy/")
            packages.remove("loopy")

        with pipargs("--no-deps"):
            for p in packages:
                install(p+"/")
                sys.path.append(os.getcwd() + "/" + p)

        # Work around easy-install.pth bug.
        try:
            packages.remove("petsc")
        except ValueError:
            pass
        packages.remove("petsc4py")
        packages.remove("firedrake")

        build_update_script()

else:
    # Update mode
    os.chdir("src")

    if args.update_script:
        # Pull firedrake, rebuild update script, launch new script
        git_update("firedrake")
        build_update_script()
        os.execv(sys.executable, [sys.executable, "../bin/firedrake-update", "--no-update-script"] + sys.argv[1:])

    deps = OrderedDict()
    deps.update(list_cloned_dependencies("PyOP2"))
    deps.update(list_cloned_dependencies("firedrake"))
    for p in options["packages"]:
        name = split_requirements_url(p)[0]
        deps.update(list_cloned_dependencies(name))
        deps[name] = p
    packages = list(deps.keys())
    packages += ["firedrake"]

    # update packages.
    if not args.honour_petsc_dir:
        petsc_changed = git_update("petsc", deps["petsc"])
    else:
        petsc_changed = False
    petsc4py_changed = git_update("petsc4py", deps["petsc4py"])

    packages.remove("petsc")
    packages.remove("petsc4py")

    if args.clean:
        clean_obsolete_packages()
        for package in packages:
            pip_uninstall(package)
        if args.rebuild:
            pip_uninstall("petsc")
            pip_uninstall("petsc4py")

    for p in packages:
        try:
            git_update(p, deps.get(p, None))
        except OSError as e:
            if e.errno == 2:
                log.warning("%s missing, cloning anew.\n" % p)
                git_clone(deps[p])
            else:
                raise

    # update dependencies.
    for p in packages:
        pip_requirements(p)

    with pipargs("--no-deps"):
        with environment(**compiler_env):
            # Only rebuild petsc if it has changed.
            if not args.honour_petsc_dir and (args.rebuild or petsc_changed):
                clean("petsc/")
                log.info("Depending on your platform, PETSc may take an hour or more to build!")
                install("petsc/")
            if args.rebuild or petsc_changed or petsc4py_changed:
                clean("petsc4py/")
                install("petsc4py/")

            # Always rebuild h5py.
            build_and_install_h5py()
            build_and_install_libspatialindex()
            build_and_install_libsupermesh()

    if "loopy" in packages:
        # We do want to install loopy's dependencies.
        clean("loopy")
        install("loopy/")
        packages.remove("loopy")
    with pipargs("--no-deps"):
        try:
            packages.remove("PyOP2")
            packages.remove("firedrake")
        except ValueError:
            pass
        packages += ("PyOP2", "firedrake")
        for p in packages:
            clean(p)
            install(p+"/")

        # Ensure pytest is at the latest version
        run_pip(["install", "-U", "pytest"])

with environment(**compiler_env):
    with pipargs("--no-deps"):
        if options["slepc"]:
            build_and_install_slepc()

        if options["opencascade"]:
            build_and_install_pythonocc()

    if args.documentation_dependencies:
        install_documentation_dependencies()

try:
    import firedrake_configuration
    firedrake_configuration.write_config(config)
    log.info("Configuration saved to configuration.json")
except Exception as e:
    log.warning("Unable to save configuration to a JSON file")
    log.warning("Error Message:")
    log.warning(str(e))

if mode == "update":
    try:
        firedrake_configuration.setup_cache_dirs()
        log.info("Clearing just in time compilation caches.")
        from firedrake.tsfc_interface import clear_cache, TSFCKernel
        from pyop2.compilation import clear_cache as pyop2_clear_cache
        print('Removing cached TSFC kernels from %s' % TSFCKernel._cachedir)
        clear_cache()
        pyop2_clear_cache()
    except:                     # noqa: E722
        # Unconditional except in order to avoid upgrade script failures.
        log.error("Failed to clear caches. Try running firedrake-clean.")


os.chdir("../..")

if mode == "install":
    log.info("\n\nSuccessfully installed Firedrake.\n")

    log.info("\nFiredrake has been installed in a python venv. You activate it with:\n")
    log.info("  . %s/bin/activate\n" % firedrake_env)
    log.info("The venv can be deactivated by running:\n")
    log.info("  deactivate\n\n")
    log.info("To upgrade Firedrake activate the venv and run firedrake-update\n")
else:
    log.info("\n\nSuccessfully updated Firedrake.\n")
