#! /usr/bin/env python3
import logging
import platform
import subprocess
import sys
import os
import shutil
from glob import glob
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from collections import OrderedDict
from distutils import dir_util
import atexit

adjoint_deps = {
    "libadjoint": "git+ssh://bitbucket.org/dolfin-adjoint/libadjoint.git",
    "dolfin-adjoint": "git+ssh://bitbucket.org/dolfin-adjoint/dolfin-adjoint.git"
}

# Packages which we wish to ensure are always recompiled
wheel_blacklist = ["mpi4py"]

# Firedrake application installation shortcuts.
firedrake_apps = {
    "gusto": ("""Atmospheric dynamical core library. http://firedrakeproject.org/gusto""",
              "git+ssh://github.com/firedrakeproject/gusto#egg=gusto"),
    "thetis": ("""Coastal ocean model. http://thetisproject.org""",
               "git+ssh://github.com/thetisproject/thetis#egg=thetis"),
    "pyadjoint": ("""New generation adjoint. https://bitbucket.org/dolfin-adjoint/pyadjoint""",
                  "git+ssh://bitbucket.org/dolfin-adjoint/pyadjoint.git#egg=pyadjoint"),
    "icepack": ("""Glacier and ice sheet model. https://icepack.github.io""",
                "git+ssh://github.com/icepack/icepack.git#egg=icepack"),
}


class InstallError(Exception):
    # Exception for generic install problems.
    pass


class FiredrakeConfiguration(dict):
    """A dictionary extended to facilitate the storage of Firedrake
    configuration information."""
    def __init__(self, args=None):
        super(FiredrakeConfiguration, self).__init__()

        '''A record of the persistent options in force.'''
        self["options"] = {}
        '''Relevant environment variables.'''
        self["environment"] = {}
        '''Additional packages installed via the plugin interface.'''
        self["additions"] = []

        if args:
            for o in self._persistent_options:
                if o in args.__dict__.keys():
                    self["options"][o] = args.__dict__[o]

    _persistent_options = ["package_manager",
                           "minimal_petsc", "mpicc", "mpicxx", "mpif90", "disable_ssh",
                           "show_petsc_configure_options", "adjoint",
                           "slepc", "slope", "packages", "honour_pythonpath",
                           "petsc_int_type", "cache_dir"]


def deep_update(this, that):
    import collections
    for k, v in that.items():
        if isinstance(v, collections.Mapping) and k in this.keys():
            this[k] = deep_update(this.get(k, {}), v)
        else:
            this[k] = v
    return this


if os.path.basename(__file__) == "firedrake-install":
    mode = "install"
elif os.path.basename(__file__) == "firedrake-update":
    mode = "update"
    os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../..")
else:
    sys.exit("Script must be invoked either as firedrake-install or firedrake-update")


if sys.version_info < (3, 5):
    if mode == "install":
        print("""\nInstalling Firedrake requires Python 3, at least version 3.5.
You should run firedrake-install with python3.""")
    if mode == "update":
        if hasattr(sys, "real_prefix"):
            # sys.real_prefix exists iff we are in an active virtualenv.
            #
            # Existing install trying to update past the py2/py3 barrier
            print("""\nFiredrake is now Python 3 only.  You cannot upgrade your existing installation.
Please follow the instructions at http://www.firedrakeproject.org/download.html to reinstall.""")
            sys.exit(1)
        else:
            # Accidentally (?) running firedrake-update with python2.
            print("""\nfiredrake-update must be run with Python 3, did you accidentally use Python 2?""")
    sys.exit(1)


branches = {}

jenkins = "JENKINS_URL" in os.environ
ci_testing_firedrake = "FIREDRAKE_CI_TESTS" in os.environ


if mode == "install":
    # Handle command line arguments.
    parser = ArgumentParser(description="""Install firedrake and its dependencies.""",
                            epilog="""The install process has three steps.

1. Any required system packages are installed using brew (MacOS) or apt (Ubuntu
   and similar Linux systems). On a Linux system without apt, the installation
   will fail if a dependency is not found.

2. A set of standard and/or third party Python packages is installed to the
   specified install location.

3. The core set of Python packages is downloaded to ./firedrake/src/ and
   installed to the specified location.

The install creates a venv in ./firedrake and installs inside
that venv.

The installer will ensure that the required configuration options are
passed to PETSc. In addition, any configure options which you provide
in the PETSC_CONFIGURE_OPTIONS environment variable will be
honoured.""",
                            formatter_class=RawDescriptionHelpFormatter)

    parser.add_argument("--adjoint", action='store_true',
                        help="Also install firedrake-adjoint.")
    parser.add_argument("--slope", action='store_true',
                        help="Also install SLOPE. This enables loop tiling in PyOP2.")
    parser.add_argument("--slepc", action="store_true",
                        help="Install SLEPc along with PETSc.")
    parser.add_argument("--disable-ssh", action="store_true",
                        help="Do not attempt to use ssh to clone git repositories: fall immediately back to https.")
    parser.add_argument("--no-package-manager", action='store_false', dest="package_manager",
                        help="Do not attempt to use apt or homebrew to install operating system packages on which we depend.")
    parser.add_argument("--minimal-petsc", action="store_true",
                        help="Minimise the set of petsc dependencies installed. This creates faster build times (useful for build testing).")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--honour-petsc-dir", action="store_true",
                       help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.")
    group.add_argument("--petsc-int-type", choices=["int32", "int64"],
                       default="int32", type=str,
                       help="The integer type used by PETSc.  Use int64 if you need to solve problems with more than 2 billion degrees of freedom.  Only takes effect if firedrake-install builds PETSc.")

    parser.add_argument("--honour-pythonpath", action="store_true",
                        help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
    parser.add_argument("--rebuild-script", action="store_true",
                        help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
    parser.add_argument("--package-branch", type=str, nargs=2, action="append", metavar=("PACKAGE", "BRANCH"),
                        help="Specify which branch of a package to use. This takes two arguments, the package name and the branch.")
    parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
    parser.add_argument("--mpicc", type=str,
                        action="store", default="mpicc",
                        help="C compiler to use when building with MPI (default is 'mpicc')")
    parser.add_argument("--mpicxx", type=str,
                        action="store", default="mpicxx",
                        help="C++ compiler to use when building with MPI (default is 'mpicxx')")
    parser.add_argument("--mpif90", type=str,
                        action="store", default="mpif90",
                        help="Fortran compiler to use when building with MPI (default is 'mpif90')")
    parser.add_argument("--show-petsc-configure-options", action="store_true",
                        help="Print out the configure options passed to PETSc")
    parser.add_argument("--venv-name", default="firedrake",
                        type=str, action="store",
                        help="Name of the venv to create (default is 'firedrake')")
    parser.add_argument("--install", action="append", dest="packages",
                        help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir . Some additional packages have shortcut install options for more information see --install-help.")
    parser.add_argument("--install-help", action="store_true",
                        help="Provide information on packages which can be installed using shortcut names.")
    parser.add_argument("--cache-dir", type=str,
                        action="store",
                        help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).")

    args = parser.parse_args()

    if args.package_branch:
        branches = {package: branch for package, branch in args.package_branch}

    args.prefix = False  # Disabled as untested
    args.packages = args.packages or []

    config = FiredrakeConfiguration(args)
else:
    import firedrake_configuration
    config = firedrake_configuration.get_config()
    if config is None:
        raise InstallError("Failed to find existing Firedrake configuration")

    parser = ArgumentParser(description="""Update this firedrake install to the latest versions of all packages.""",
                            formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument("--no-update-script", action="store_false", dest="update_script",
                        help="Do not update script before updating Firedrake.")
    parser.add_argument("--rebuild", action="store_true",
                        help="Rebuild all packages even if no new version is available. Usually petsc and petsc4py are only rebuilt if they change. All other packages are always rebuilt.")
    parser.add_argument("--rebuild-script", action="store_true",
                        help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
    parser.add_argument("--adjoint", action='store_true', dest="adjoint", default=config["options"]["adjoint"],
                        help="Also install firedrake-adjoint.")
    parser.add_argument("--slope", action='store_true', dest="slope", default=config["options"]["slope"],
                        help="Also install SLOPE. This enables loop tiling in PyOP2.")
    parser.add_argument("--slepc", action="store_true", dest="slepc", default=config["options"]["slepc"],
                        help="Install SLEPc along with PETSc")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--honour-petsc-dir", action="store_true",
                       help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.")
    group.add_argument("--petsc-int-type", choices=["int32", "int64"],
                       default="int32", type=str,
                       help="The integer type used by PETSc.  Use int64 if you need to solve problems with more than 2 billion degrees of freedom.  Only takes effect if firedrake-install builds PETSc.")

    parser.add_argument("--honour-pythonpath", action="store_true", default=config["options"].get("honour_pythonpath", False),
                        help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
    parser.add_argument("--clean", action='store_true',
                        help="Delete any remnants of obsolete Firedrake components.")
    parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
    parser.add_argument("--install", action="append", dest="packages", default=config["options"].get("packages", []),
                        help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir. Some additional packages have shortcut install options for more information see --install-help.")
    parser.add_argument("--install-help", action="store_true",
                        help="Provide information on packages which can be installed using shortcut names.")
    parser.add_argument("--cache-dir", type=str,
                        action="store", default=config["options"].get("cache_dir", ""),
                        help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).")

    args = parser.parse_args()

    args.packages = list(set(args.packages))  # remove duplicates

    petsc_int_type_changed = False
    if config["options"].get("petsc_int_type", "int32") != args.petsc_int_type:
        petsc_int_type_changed = True
        args.rebuild = True

    config = deep_update(config, FiredrakeConfiguration(args))
    if "mpicxx" not in config["options"]:
        config["options"]["mpicxx"] = "mpicxx"
    if "mpif90" not in config["options"]:
        config["options"]["mpif90"] = "mpif90"

if args.install_help:
    help_string = """
You can install the following packages by passing --install shortname
where shortname is one of the names given below:

"""
    componentformat = "|{:10}|{:70}|\n"
    header = componentformat.format("Name", "Description")
    line = "-" * (len(header) - 1) + "\n"
    help_string += line + header + line
    for package, d in firedrake_apps.items():
        help_string += componentformat.format(package, d[0])
    help_string += line
    print(help_string)
    sys.exit(0)

# Set up logging
# Log to file at DEBUG level
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(levelname)-6s %(message)s',
                    filename='firedrake-%s.log' % mode,
                    filemode='w')
# Log to console at INFO level
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logging.getLogger().addHandler(console)

log = logging.getLogger()

log.info("Running %s" % " ".join(sys.argv))


@atexit.register
def print_log_location():
    log.info("\n\n%s log saved in firedrake-%s.log" % (mode.capitalize(), mode))


class directory(object):
    """Context manager that executes body in a given directory"""
    def __init__(self, dir):
        self.dir = os.path.abspath(dir)

    def __enter__(self):
        self.olddir = os.path.abspath(os.getcwd())
        log.debug("Old path '%s'" % self.olddir)
        log.debug("Pushing path '%s'" % self.dir)
        os.chdir(self.dir)

    def __exit__(self, *args):
        log.debug("Popping path '%s'" % self.dir)
        os.chdir(self.olddir)
        log.debug("New path '%s'" % self.olddir)


options = config["options"]

# Apply short cut package names.
options["packages"] = [firedrake_apps.get(p, (None, p))[1] for p in options["packages"]]


# Record of obsolete packages which --clean should remove from old installs.
old_git_packages = []

if mode == "install":
    firedrake_env = os.path.abspath(args.venv_name)
else:
    try:
        firedrake_env = os.environ["VIRTUAL_ENV"]
    except KeyError:
        quit("Unable to retrieve venv name from the environment.\n Please ensure the venv is active before running firedrake-update.")

if "cache_dir" not in config["options"] or not config["options"]["cache_dir"]:
    config["options"]["cache_dir"] = os.path.join(firedrake_env, ".cache")


# venv install
python = ["%s/bin/python" % firedrake_env]
# Use the pip from the venv
pip = python + ["-m", "pip"]
pipinstall = pip + ["install", "--no-binary", ",".join(wheel_blacklist)]
pyinstall = python + ["setup.py", "install"]

if "PYTHONPATH" in os.environ and not args.honour_pythonpath:
    quit("""The PYTHONPATH environment variable is set. This is probably an error.
If you really want to use your own Python packages, please run again with the
--honour-pythonpath option.
""")

petsc_opts = "--download-eigen=%s/src/eigen-3.3.3.tgz " % firedrake_env

petsc_opts += "--with-fortran-bindings=0 "

if options["minimal_petsc"]:
    if options["petsc_int_type"] == "int64":
        petsc_opts += """--download-metis --download-parmetis --download-hdf5 --download-hypre --with-64-bit-indices"""
    else:
        petsc_opts += """--download-chaco --download-metis --download-parmetis --download-scalapack --download-hypre --download-mumps --download-hdf5"""
else:
    if options["petsc_int_type"] == "int64":
        petsc_opts += """--download-metis --download-parmetis --download-hypre --download-netcdf --download-pnetcdf --download-hdf5 --download-exodusii --with-64-bit-indices"""
    else:
        petsc_opts += """--download-chaco --download-metis --download-parmetis --download-scalapack --download-hypre --download-mumps --download-netcdf --download-hdf5 --download-pnetcdf --download-exodusii"""


if "PETSC_CONFIGURE_OPTIONS" not in os.environ:
    os.environ["PETSC_CONFIGURE_OPTIONS"] = petsc_opts
else:
    for opt in petsc_opts.split():
        if opt not in os.environ["PETSC_CONFIGURE_OPTIONS"]:
            os.environ["PETSC_CONFIGURE_OPTIONS"] += " " + opt

if mode == "update" and petsc_int_type_changed:
    log.warning("""Force rebuilding all packages because PETSc int type changed""")


def check_call(arguments, env=None):
    try:
        log.debug("Running command '%s'", " ".join(arguments))
        log.debug(subprocess.check_output(arguments, stderr=subprocess.STDOUT, env=env).decode())
    except subprocess.CalledProcessError as e:
        log.debug(e.output.decode())
        raise


def check_output(args, env=None):
    try:
        log.debug("Running command '%s'", " ".join(args))
        result = subprocess.check_output(args, stderr=subprocess.STDOUT, env=env)
        if isinstance(result, str):
            # Python 2
            return result
        else:
            # Python 3
            return result.decode()
    except subprocess.CalledProcessError as e:
        log.debug(e.output.decode())
        raise


def brew_install(name, options=None):
    try:
        # Check if it's already installed
        check_call(["brew", "list", name])
    except subprocess.CalledProcessError:
        # If not found, go ahead and install
        arguments = [name]
        if options:
            arguments = options + arguments
        if args.verbose:
            arguments = ["--verbose"] + arguments
        check_call(["brew", "install"] + arguments)


def apt_check(name):
    log.info("Checking for presence of package %s..." % name)
    # Note that subprocess return codes have the opposite logical
    # meanings to those of Python variables.
    try:
        check_call(["dpkg-query", "-s", name])
        log.info("  installed.")
        return True
    except subprocess.CalledProcessError:
        log.info("  missing.")
        return False


def apt_install(names):
    log.info("Installing missing packages: %s." % ", ".join(names))
    if sys.stdin.isatty():
        subprocess.check_call(["sudo", "apt-get", "install"] + names)
    else:
        log.info("Non-interactive stdin detected; installing without prompts")
        subprocess.check_call(["sudo", "apt-get", "-y", "install"] + names)


def split_requirements_url(url):
    name = url.split(".git")[0].split("#")[0].split("/")[-1]
    spliturl = url.split("://")[1].split("#")[0].split("@")
    try:
        plain_url, branch = spliturl
    except ValueError:
        plain_url = spliturl[0]
        branch = "master"
    return name, plain_url, branch


def git_url(plain_url, protocol):
    if protocol == "ssh":
        return "git@%s:%s" % tuple(plain_url.split("/", 1))
    elif protocol == "https":
        return "https://%s" % plain_url
    else:
        raise ValueError("Unknown git protocol: %s" % protocol)


def git_clone(url):
    name, plain_url, branch = split_requirements_url(url)
    if name == "petsc" and args.honour_petsc_dir:
        log.info("Using existing petsc installation\n")
        return name
    log.info("Cloning %s\n" % name)
    if name in branches:
        branch = branches[name]
    try:
        if options["disable_ssh"]:
            raise InstallError("Skipping ssh clone because --disable-ssh")
        check_call(["git", "clone", "-q", git_url(plain_url, "ssh")])
        log.info("Successfully cloned repository %s" % name)
    except (subprocess.CalledProcessError, InstallError):
        if not options["disable_ssh"]:
            log.warn("Failed to clone %s using ssh, falling back to https." % name)
        try:
            check_call(["git", "clone", "-q", git_url(plain_url, "https")])
            log.info("Successfully cloned repository %s." % name)
        except subprocess.CalledProcessError:
            log.error("Failed to clone %s branch %s." % (name, branch))
            raise
    with directory(name):
        try:
            log.info("Checking out branch %s" % branch)
            check_call(["git", "checkout", "-q", branch])
            log.info("Successfully checked out branch %s" % branch)
        except subprocess.CalledProcessError:
            log.error("Failed to check out branch %s" % branch)
            raise
    return name


def list_cloned_dependencies(name):
    log.info("Finding dependencies of %s\n" % name)
    deps = OrderedDict()
    try:
        for dep in open(name + "/requirements-git.txt", "r"):
            name = split_requirements_url(dep.strip())[0]
            deps[name] = dep.strip()
    except IOError:
        pass
    return deps


def clone_dependencies(name):
    log.info("Cloning the dependencies of %s" % name)
    deps = []
    try:
        for dep in open(name + "/requirements-git.txt", "r"):
            deps.append(git_clone(dep.strip()))
    except IOError:
        pass
    return deps


def git_update(name, url=None):
    # Update the named git repo and return true if the current branch actually changed.
    log.info("Updating the git repository for %s" % name)
    with directory(name):
        git_sha = check_output(["git", "rev-parse", "HEAD"])
        # Ensure remotes get updated if and when we move repositories.
        if url:
            plain_url = split_requirements_url(url)[1]
            current_url = check_output(["git", "remote", "-v"]).split()[1]
            protocol = "https" if current_url.startswith("https") else "ssh"
            new_url = git_url(plain_url, protocol)
            # Ensure we only change from bitbucket to github and not the reverse.
            if new_url != current_url and "bitbucket.org" in current_url \
               and "github.com/firedrakeproject" in plain_url:
                log.info("Updating git remote for %s" % name)
                check_call(["git", "remote", "set-url", "origin", new_url])
        check_call(["git", "pull"])
        git_sha_new = check_output(["git", "rev-parse", "HEAD"])
    return git_sha != git_sha_new


def run_pip(args):
    check_call(pip + args)


def run_pip_install(pipargs):
    # Make pip verbose when logging, so we see what the
    # subprocesses wrote out.
    # Particularly important for debugging petsc fails.
    pipargs = ["-vvv"] + pipargs
    check_call(pipinstall + pipargs)


def run_cmd(args):
    check_call(args)


def pip_requirements(package):
    log.info("Installing pip dependencies for %s" % package)
    if os.path.isfile("%s/requirements-ext.txt" % package):
        run_pip_install(["-r", "%s/requirements-ext.txt" % package])
    elif os.path.isfile("%s/requirements.txt" % package):
        if package == "COFFEE":
            # FIXME: Horrible hack to work around
            # https://github.com/coin-or/pulp/issues/123
            run_pip_install(["--no-deps", "-r", "%s/requirements.txt" % package])
        else:
            run_pip_install(["-r", "%s/requirements.txt" % package])
    else:
        log.info("No dependencies found. Skipping.")


def install(package):
    log.info("Installing %s" % package)
    if package == "petsc/":
        build_and_install_petsc()
    # The following outrageous hack works around the fact that petsc and co. cannot be installed in developer mode.
    elif package not in ["petsc4py/", "slepc/", "slepc4py/"]:
        run_pip_install(["-e", package])
    else:
        run_pip_install(["--ignore-installed", package])


def clean(package):
    log.info("Cleaning %s" % package)
    with directory(package):
        check_call(["python", "setup.py", "clean"])


def pip_uninstall(package):
    log.info("Removing existing %s installations\n" % package)
    # Uninstalling something with pip is an absolute disaster.  We
    # have to use pip freeze to list all available packages "locally"
    # and keep on removing the one we want until it is gone from this
    # list!  Yes, pip will happily have two different versions of the
    # same package co-existing.  Moreover, depending on the phase of
    # the moon, the order in which they are uninstalled is not the
    # same as the order in which they appear on sys.path!
    again = True
    i = 0
    while again:
        # List installed packages, "locally".  In a venv,
        # this just tells me packages in the venv, otherwise it
        # gives me everything.
        lines = check_output(pip + ["freeze", "-l"])
        again = False
        for line in lines.split("\n"):
            # Do we have a locally installed package?
            if line.startswith(package):
                # Uninstall it.
                run_pip(["uninstall", "-y", line.strip()])
                # Go round again, because THERE MIGHT BE ANOTHER ONE!
                again = True
        i += 1
        if i > 10:
            raise InstallError("pip claims it uninstalled %s more than 10 times.  Something is probably broken.", package)


def get_petsc_dir():
    if args.honour_petsc_dir:
        try:
            petsc_dir = os.environ["PETSC_DIR"]
        except KeyError:
            raise InstallError("Unable to find installed PETSc (did you forget to set PETSC_DIR?)")
        petsc_arch = os.environ.get("PETSC_ARCH", "")
    else:
        try:
            petsc_dir = check_output(python +
                                     ["-c", "import petsc; print(petsc.get_petsc_dir())"]).strip()

            petsc_arch = ""
        except subprocess.CalledProcessError:
            raise InstallError("Unable to find installed PETSc")
    return petsc_dir, petsc_arch


def get_slepc_dir():
    petsc_dir, petsc_arch = get_petsc_dir()
    if args.honour_petsc_dir:
        try:
            slepc_dir = os.environ["SLEPC_DIR"]
        except KeyError:
            raise InstallError("Need to set SLEPC_DIR for --slepc with --honour-petsc-dir")
    else:
        try:
            slepc_dir = check_output(python +
                                     ["-c", "import slepc; print(slepc.get_slepc_dir())"]).strip()
        except subprocess.CalledProcessError:
            raise InstallError("Unable to find installed SLEPc")
    return slepc_dir, petsc_arch


def build_and_install_petsc():
    import hashlib
    import urllib.request
    tarball = "eigen-3.3.3.tgz"
    url = "https://bitbucket.org/eigen/eigen/get/3.3.3.tar.gz"
    sha = hashlib.sha256()
    expect = "94878cbfa27b0d0fbc64c00d4aafa137f678d5315ae62ba4aecddbd4269ae75f"
    if not os.path.exists(tarball):
        log.info("Downloading Eigen from '%s' to '%s'" % (url, tarball))
        urllib.request.urlretrieve(url, filename=tarball)
    else:
        log.info("Eigen tarball already downloaded to '%s'" % tarball)
    log.info("Checking Eigen tarball integrity")
    with open(tarball, "rb") as f:
        while True:
            data = f.read(65536)
            if not data:
                break
            sha.update(data)
    actual = sha.hexdigest()
    if actual != expect:
        raise InstallError("Downloaded Eigen tarball has incorrect sha256sum, expected '%s', was '%s'",
                           expect, actual)
    else:
        log.info("Eigen tarball hash valid")
    log.info("Building PETSc. \nDepending on your platform, may take between a few minutes and an hour or more to build!")
    os.environ["MPICC"] = options["mpicc"] or "mpicc"
    os.environ["MPICXX"] = options["mpicxx"] or "mpicxx"
    os.environ["MPIF90"] = options["mpif90"] or "mpif90"
    run_pip_install(["--ignore-installed", "petsc/"])


def build_and_install_h5py():
    import shutil
    log.info("Installing h5py")
    # Clean up old downloads
    if os.path.exists("h5py-2.5.0"):
        log.info("Removing old h5py-2.5.0 source")
        shutil.rmtree("h5py-2.5.0")
    if os.path.exists("h5py.tar.gz"):
        log.info("Removing old h5py.tar.gz")
        os.remove("h5py.tar.gz")

    url = "git+https://github.com/firedrakeproject/h5py.git@firedrake#egg=h5py"
    if os.path.exists("h5py"):
        changed = False
        with directory("h5py"):
            # Rewrite old h5py/h5py remote to firedrakeproject/h5py remote.
            plain_url = split_requirements_url(url)[1]
            current_remote = check_output(["git", "remote", "-v"]).split()[1]
            proto = "https" if current_remote.startswith("https") else "ssh"
            new_remote = git_url(plain_url, proto)
            if new_remote != current_remote and "h5py/h5py.git" in current_remote:
                log.info("Updating git remote for h5py from %s to %s", current_remote, new_remote)
                check_call(["git", "remote", "set-url", "origin", new_remote])
                check_call(["git", "fetch", "-p", "origin"])
                check_call(["git", "checkout", "firedrake"])
                changed = True
        changed |= git_update("h5py")
    else:
        git_clone(url)
        changed = True

    petsc_dir, petsc_arch = get_petsc_dir()
    hdf5_dir = "%s/%s" % (petsc_dir, petsc_arch)
    if changed or args.rebuild:
        log.info("Linking h5py against PETSc found in %s\n" % hdf5_dir)
        oldcc = os.environ.get("CC", None)
        os.environ["CC"] = options["mpicc"] or "mpicc"
        os.environ["HDF5_DIR"] = hdf5_dir
        os.environ["HDF5_MPI"] = "ON"
        # Only uninstall if things changed.
        pip_uninstall("h5py")
        # Pip installing from dirty directory is potentially unsafe.
        with directory("h5py/"):
            check_call(["git", "clean", "-fdx"])
        install("h5py/")
        if oldcc is None:
            del os.environ["CC"]
        else:
            os.environ["CC"] = oldcc
    else:
        log.info("No need to rebuild h5py")


def build_and_install_libspatialindex():
    log.info("Installing libspatialindex")
    if os.path.exists("libspatialindex"):
        log.info("Updating the git repository for libspatialindex")
        with directory("libspatialindex"):
            check_call(["git", "fetch"])
            git_sha = check_output(["git", "rev-parse", "HEAD"])
            git_sha_new = check_output(["git", "rev-parse", "@{u}"])
            libspatialindex_changed = git_sha != git_sha_new
            if libspatialindex_changed:
                check_call(["git", "reset", "--hard"])
                check_call(["git", "pull"])
    else:
        git_clone("git+https://github.com/firedrakeproject/libspatialindex.git")
        libspatialindex_changed = True

    if libspatialindex_changed:
        with directory("libspatialindex"):
            # Clean source directory
            check_call(["git", "reset", "--hard"])
            check_call(["git", "clean", "-f", "-x", "-d"])
            # Patch Makefile.am to skip building test
            check_call(["sed", "-i", "-e", "/^SUBDIRS/s/ test//", "Makefile.am"])
            # Build and install
            check_call(["./autogen.sh"])
            check_call(["./configure", "--prefix=" + firedrake_env,
                        "--enable-shared", "--disable-static"])
            check_call(["make"])
            check_call(["make", "install"])
    else:
        log.info("No need to rebuild libspatialindex")


def build_and_install_slepc():
    petsc_dir, petsc_arch = get_petsc_dir()

    if args.honour_petsc_dir:
        slepc_dir, slepc_arch = get_slepc_dir()
        log.info("Using installed SLEPc from %s/%s", slepc_dir, slepc_arch)
    else:
        log.info("Installing SLEPc.")
        url = "git+https://github.com/firedrakeproject/slepc.git@firedrake"
        if os.path.exists("slepc"):
            slepc_changed = False
            with directory("slepc"):
                plain_url = split_requirements_url(url)[1]
                current_remote = check_output(["git", "remote", "-v"]).split()[1]
                proto = "https" if current_remote.startswith("https") else "ssh"
                new_remote = git_url(plain_url, proto)
                if new_remote != current_remote:
                    log.info("Updating git remote for SLEPc from %s to %s", current_remote, new_remote)
                    check_call(["git", "remote", "set-url", "origin", new_remote])
                    check_call(["git", "fetch", "-p", "origin"])
                    check_call(["git", "checkout", "firedrake"])
                    slepc_changed = True
            slepc_changed |= git_update("slepc")
        else:
            git_clone(url)
            slepc_changed = True
        if slepc_changed:
            install("slepc/")
        else:
            log.info("No need to rebuild SLEPc")

    log.info("Installing slepc4py.")
    url = "git+https://github.com/firedrakeproject/slepc4py.git@firedrake"
    if os.path.exists("slepc4py"):
        slepc4py_changed = False
        with directory("slepc4py"):
                plain_url = split_requirements_url(url)[1]
                current_remote = check_output(["git", "remote", "-v"]).split()[1]
                proto = "https" if current_remote.startswith("https") else "ssh"
                new_remote = git_url(plain_url, proto)
                if new_remote != current_remote:
                    log.info("Updating git remote for slepc4py from %s to %s", current_remote, new_remote)
                    check_call(["git", "remote", "set-url", "origin", new_remote])
                    check_call(["git", "fetch", "-p", "origin"])
                    check_call(["git", "checkout", "firedrake"])
                    slepc4py_changed = True
        slepc4py_changed |= git_update("slepc4py")
    else:
        git_clone(url)
        slepc4py_changed = True
    if slepc4py_changed:
        install("slepc4py/")
    else:
        log.info("No need to rebuild slepc4py")


def build_and_install_adjoint():

    for package in adjoint_deps:
        try:
            git_update(package, adjoint_deps[package])
        except OSError as e:
            if e.errno == 2:
                log.warn("%s missing, cloning anew.\n" % package)
                git_clone(adjoint_deps[package])
            else:
                raise

    env = dict(os.environ)
    petsc_dir, petsc_arch = get_petsc_dir()
    env["PETSC_ARCH"] = petsc_arch
    env["PETSC_DIR"] = petsc_dir
    if options["slepc"]:
        slepc_dir, _ = get_slepc_dir()
        env["SLEPC_DIR"] = slepc_dir

    with directory("libadjoint"):
        dir_util.mkpath("build")
        with directory("build"):
            log.info("Configuring libadjoint for venv installation in %s" % os.environ["VIRTUAL_ENV"])
            check_call(["cmake", "-DCMAKE_INSTALL_PREFIX=%s" % os.environ["VIRTUAL_ENV"],
                        "-DPYTHON_EXECUTABLE=%s/bin/python3" % os.environ["VIRTUAL_ENV"], ".."], env=env)
            log.info("Installing libadjoint.")
            check_call(["make", "install"])

    install("dolfin-adjoint/")


def build_and_install_slope():
    log.info("Installing SLOPE")
    if os.path.exists("SLOPE"):
        slope_changed = git_update("SLOPE")
    else:
        git_clone("git+https://github.com/coneoproject/SLOPE.git")
        slope_changed = True

    if slope_changed:
        with directory("SLOPE"):
            # Clean source directory
            check_call(["git", "reset", "--hard"])
            check_call(["git", "clean", "-f", "-x", "-d"])
            # Build and install (hack: need to do this manually as SLOPE is
            # still stupid and doesn't have a proper configure+make+install system)
            check_call(["make"])
            check_call(["mv"] + glob(os.path.join('lib', '*')) + [os.path.join(firedrake_env, 'lib')])
            include_dir = os.path.join(firedrake_env, 'include', 'SLOPE')
            check_call(["mkdir", "-p", include_dir])
            check_call(["cp"] + glob(os.path.join('sparsetiling', 'include', '*')) + [include_dir])
            # Also install the Python interface
            run_cmd(pyinstall)
    else:
        log.info("No need to rebuild SLOPE")


def clean_obsolete_packages():
    dirs = os.listdir(".")
    for package in old_git_packages:
        pip_uninstall(package)
        if package in dirs:
            shutil.rmtree(package)


def quit(message):
    log.error(message)
    sys.exit(1)


def build_update_script():
    log.info("Creating firedrake-update script.")
    with open("firedrake/scripts/firedrake-install", "r") as f:
        update_script = f.read()

    try:
        os.mkdir("../bin")
    except OSError:
        pass
    with open("../bin/firedrake-update", "w") as f:
        f.write(update_script)
    check_call(["chmod", "a+x", "../bin/firedrake-update"])


if options["show_petsc_configure_options"]:
    log.info("******************************************\n")
    log.info("Building PETSc with the following options:\n")
    log.info("******************************************\n")
    log.info("%s\n\n" % os.environ["PETSC_CONFIGURE_OPTIONS"])

if args.rebuild_script:
    os.chdir(os.path.dirname(os.path.realpath(__file__)) + ("/../.."))

    build_update_script()

    log.info("Successfully rebuilt firedrake-update.\n")

    log.info("To upgrade firedrake, run firedrake-update")
    sys.exit(0)


if "PETSC_DIR" in os.environ and not args.honour_petsc_dir:
    quit("""The PETSC_DIR environment variable is set. This is probably an error.
If you really want to use your own PETSc build, please run again with the
--honour-petsc-dir option.
""")

if "PETSC_DIR" not in os.environ and args.honour_petsc_dir:
    quit("""The --honour-petsc-dir is set, but PETSC_DIR environment variable is
not defined. If you have compiled PETSc manually, set PETSC_DIR
(and optionally PETSC_ARCH) variables to point to the build directory.
""")

if "SLEPC_DIR" not in os.environ and args.honour_petsc_dir and options["slepc"]:
    quit("""If you use --honour-petsc-dir, you must also build SLEPc manually
and set the SLEPC_DIR environment variable appropriately""")

if "SLEPC_DIR" in os.environ and not args.honour_petsc_dir and options["slepc"]:
    quit("""The SLEPC_DIR environment variable is set.  If you want to use
your own SLEPc version, you must also build your own PETSc and run
with --honour-petsc-dir.""")

log.debug("*** Current environment (output of 'env') ***")
log.debug(check_output(["env"]))
log.debug("\n\n")

if mode == "install" or not args.update_script:
    # Check operating system.
    osname = platform.uname()[0]
    if osname == "Darwin":
        if options["package_manager"]:

            log.info("Installing command line tools...")
            try:
                check_call(["xcode-select", "--install"])
            except subprocess.CalledProcessError:
                # expected failure if already installed
                pass

            try:
                check_call(["brew", "--version"])
            except subprocess.CalledProcessError:
                quit("Homebrew not found. Please install it using the instructions at http://brew.sh and then try again.")

            log.info("Installing required packages via homebrew. You can safely ignore warnings that packages are already installed")
            # Ensure a fortran compiler is available
            brew_install("gcc")
            brew_install("openmpi")
            brew_install("python3")
            brew_install("autoconf")
            brew_install("automake")
            brew_install("cmake")
            brew_install("libtool")
            brew_install("mercurial")

        else:
            log.info("Xcode and homebrew installation disabled. Proceeding on the rash assumption that packaged dependencies are in place.")

    elif osname == "Linux":
        # Check for apt.
        try:
            if not options["package_manager"]:
                raise InstallError
            check_call(["apt-get", "--version"])

            apt_packages = ["build-essential",
                            "autoconf",
                            "automake",
                            "cmake",
                            "gfortran",
                            "git-core",
                            "libblas-dev",
                            "liblapack-dev",
                            "libopenmpi-dev",
                            "libtool",
                            "mercurial",
                            "openmpi-bin",
                            "python3-dev",
                            "python3-pip",
                            "python3-tk",
                            "python3-venv",
                            "zlib1g-dev"]

            missing_packages = [p for p in apt_packages if not apt_check(p)]
            if missing_packages:
                apt_install(missing_packages)

        except (subprocess.CalledProcessError, InstallError):
            log.info("apt-get not found or disabled. Proceeding on the rash assumption that your compiled dependencies are in place.")
            log.info("If this is not the case, please install the following and try again:")
            log.info("* A C and C++ compiler (for example gcc/g++ or clang), GNU make")
            log.info("* A Fortran compiler (for PETSc)")
            log.info("* MPI")
            log.info("* Blas and Lapack")
            log.info("* Git, Mercurial")
            log.info("* Python version >=3.5")
            log.info("* The Python headers")
            log.info("* autoconf, automake, libtool")
            log.info("* CMake")
            log.info("* zlib")

    else:
        log.warn("You do not appear to be running Linux or MacOS. Please do not be surprised if this install fails.")


if mode == "install":
    if os.path.exists(firedrake_env):
        log.warning("Specified venv '%s' already exists", firedrake_env)
        quit("Can't install into existing venv '%s'" % firedrake_env)

    log.info("Creating firedrake venv in '%s'." % firedrake_env)
    # Debian's Python3 is screwed, they don't ship ensurepip as part
    # of the base python package, so the default python -m venv
    # doesn't work.  Moreover, they have spiked the file such that it
    # calls sys.exit, which will kill any attempts to create a venv
    # with pip.
    try:
        import ensurepip        # noqa: F401
        with_pip = True
    except ImportError:
        with_pip = False
    import venv
    venv.EnvBuilder(with_pip=with_pip).create(firedrake_env)
    if not with_pip:
        import urllib.request
        log.debug("ensurepip unavailable, bootstrapping pip using get-pip.py")
        urllib.request.urlretrieve("https://bootstrap.pypa.io/get-pip.py", filename="get-pip.py")
        check_call(python + ["get-pip.py"])
        log.debug("bootstrapping pip succeeded")
        log.debug("Removing get-pip.py")
        os.remove("get-pip.py")
    # Ensure pip and setuptools are at the latest version.
    run_pip(["install", "-U", "setuptools"])
    run_pip(["install", "-U", "pip"])

    # We haven't activated the venv so we need to manually set the environment.
    os.environ["VIRTUAL_ENV"] = firedrake_env

os.chdir(firedrake_env)

if mode == "install":
    os.mkdir("src")
    os.chdir("src")

    if jenkins and ci_testing_firedrake:
        check_call(["ln", "-s", "../../../", "firedrake"])
    else:
        git_clone("git+https://github.com/firedrakeproject/firedrake.git")

    packages = clone_dependencies("firedrake")
    packages = clone_dependencies("PyOP2") + packages
    packages += ["firedrake"]

    for p in options["packages"]:
        name = git_clone(p)
        packages.extend(clone_dependencies(name))
        packages += [name]

    if args.honour_petsc_dir:
        packages.remove("petsc")

    # Force Cython to install first to work around pip dependency issues.
    run_pip_install(["Cython>=0.22"])

    # Need to install petsc first in order to resolve hdf5 dependency.
    if not args.honour_petsc_dir:
        pipinstall.append("--no-deps")
        packages.remove("petsc")
        install("petsc/")
        pipinstall.pop()

    for p in packages:
        pip_requirements(p)

    build_and_install_h5py()
    build_and_install_libspatialindex()
    pipinstall.append("--no-deps")
    for p in packages:
        install(p+"/")
        sys.path.append(os.getcwd() + "/" + p)

    # Work around easy-install.pth bug.
    try:
        packages.remove("petsc")
    except ValueError:
        pass
    packages.remove("petsc4py")
    packages.remove("firedrake")
    v = sys.version_info
    if v.major <= 2:
        easyinstall = open("../lib/python%d.%d/site-packages/easy-install.pth" % (v.major, v.minor), "r").readlines()
        new_packages = [os.getcwd() + "/" + p + "\n" for p in packages]
        open("../lib/python%d.%d/site-packages/easy-install.pth" % (v.major, v.minor), "w").writelines(
            easyinstall[:1] + new_packages + easyinstall[1:])

    build_update_script()

else:
    # Update mode
    os.chdir("src")

    if args.update_script:
        # Pull firedrake, rebuild update script, launch new script
        git_update("firedrake")
        build_update_script()
        os.execv(sys.executable, [sys.executable, "../bin/firedrake-update", "--no-update-script"] + sys.argv[1:])

    deps = OrderedDict()
    deps.update(list_cloned_dependencies("PyOP2"))
    deps.update(list_cloned_dependencies("firedrake"))
    for p in options["packages"]:
        name = split_requirements_url(p)[0]
        deps.update(list_cloned_dependencies(name))
        deps[name] = p
    packages = list(deps.keys())
    packages += ["firedrake"]

    # update packages.
    if not args.honour_petsc_dir:
        petsc_changed = git_update("petsc", deps["petsc"])
    else:
        petsc_changed = False
    petsc4py_changed = git_update("petsc4py", deps["petsc4py"])

    packages.remove("petsc")
    packages.remove("petsc4py")

    if args.clean:
        clean_obsolete_packages()
        for package in packages:
            pip_uninstall(package)
        if args.rebuild:
            pip_uninstall("petsc")
            pip_uninstall("petsc4py")

    for p in packages:
        try:
            git_update(p, deps.get(p, None))
        except OSError as e:
            if e.errno == 2:
                log.warn("%s missing, cloning anew.\n" % p)
                git_clone(deps[p])
            else:
                raise

    # update dependencies.
    for p in packages:
        pip_requirements(p)
    pipinstall.append("--no-deps")

    # Only rebuild petsc if it has changed.
    if not args.honour_petsc_dir and (args.rebuild or petsc_changed):
        clean("petsc/")
        log.info("Depending on your platform, PETSc may take an hour or more to build!")
        install("petsc/")
    if args.rebuild or petsc_changed or petsc4py_changed:
        clean("petsc4py/")
        install("petsc4py/")

    # Always rebuild h5py.
    build_and_install_h5py()
    build_and_install_libspatialindex()

    try:
        packages.remove("PyOP2")
        packages.remove("firedrake")
    except ValueError:
        pass
    packages += ("PyOP2", "firedrake")
    for p in packages:
        clean(p)
        install(p+"/")

if options["slepc"]:
    build_and_install_slepc()
if options["adjoint"]:
    build_and_install_adjoint()
if options["slope"]:
    build_and_install_slope()

try:
    import firedrake_configuration
    firedrake_configuration.write_config(config)
    log.info("Configuration saved to configuration.json")
except Exception as e:
    log.warning("Unable to save configuration to a JSON file")
    log.warning("Error Message:")
    log.warning(str(e))

if mode == "update":
    try:
        firedrake_configuration.setup_cache_dirs()
        log.info("Clearing just in time compilation caches.")
        from firedrake.tsfc_interface import clear_cache, TSFCKernel
        from pyop2.compilation import clear_cache as pyop2_clear_cache
        print('Removing cached TSFC kernels from %s' % TSFCKernel._cachedir)
        clear_cache()
        pyop2_clear_cache()
    except:                     # noqa: E722
        # Unconditional except in order to avoid upgrade script failures.
        log.error("Failed to clear caches. Try running firedrake-clean.")


os.chdir("../..")

if mode == "install":
    log.info("\n\nSuccessfully installed Firedrake.\n")

    log.info("\nFiredrake has been installed in a python venv. You activate it with:\n")
    log.info("  . %s/bin/activate\n" % firedrake_env)
    log.info("The venv can be deactivated by running:\n")
    log.info("  deactivate\n\n")
    log.info("To upgrade Firedrake activate the venv and run firedrake-update\n")
else:
    log.info("\n\nSuccessfully updated Firedrake.\n")
