#!/usr/bin/python3

# SPDX-License-Identifier: BSD-3-Clause
#
# Authors: Hugo Lefeuvre <hugo.lefeuvre@manchester.ac.uk>
#
# Copyright (c) 2020-2023, The University of Manchester. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
import argparse
import pathlib
import shutil
import git
import re
import glob
import sys
import src.common as common
import subprocess
import threading
from distutils.dir_util import copy_tree
from src.common import *
from datetime import datetime

ENABLE_DIRTY_DB = False
NUMBER_GENERATE_REPLICAS = 2
NUMBER_PARALLEL_REPLICAS = 2

ONLY_DOCKER_OPT = "--only-build-docker"

assert(NUMBER_GENERATE_REPLICAS % NUMBER_PARALLEL_REPLICAS == 0)

def open_syscall_file(path):
    with open(path, "r") as f:
        l = list(set([e for e in f.read().split("\n") if len(e) > 1]))
        l = format_syscall_list_to_num(l)
        l.sort()
        return l

# init a fresh database
def init_db(path):
    if (not os.path.isdir(path)):
        os.mkdir(path)
    g = git.cmd.Git(path)
    g.init()

# check that the db is sane
def db_check(path):
    # it must exist and be in a git repo
    try:
        git_repo = git.Repo(path, search_parent_directories=True)
    except git.exc.NoSuchPathError:
        error("Database %s does not exist" % path)
        create = input("Create it? [y/n] ")
        if create.lower() == 'yes' or create.lower() == 'y':
            init_db(path)
            return True
        else:
            return False
    except git.exc.InvalidGitRepositoryError:
        error("Database %s is not a git repository" % path)
        return False

    # it may not have uncommited changes
    git_repo = git.Repo(path, search_parent_directories=True)
    if (git_repo.is_dirty() or len(git_repo.untracked_files) != 0):
        if (not ENABLE_DIRTY_DB):
            error("Database %s is dirty; commit your changes before running this tool." % path)
            return False
        else:
            warning("DB dirty, ignoring.")

    return True

# return an in-memory representation of the DB:
# {
#   # first level: applications
#   redis : {
#     # second level: workloads
#     benchmark : {
#       # third level: individual measurements matched with the
#       # measurement mechanism. If several result sets are provided,
#       # the latest one is taken
#       dynamic : [[0, 'Y', 'N', 'Y'], [1, 'N', 'N', 'N'], ...],
#       static_binary : [[0, 'Y'], [1, 'N'], ...],
#       static_source : [[0, 'N'], [1, 'N'], ...]
#     }
#     testsuite : {
#       dynamic : [[0, 'Y', 'N', 'N'], [1, 'N', 'N', 'N'], ...],
#       static_binary : [[0, 'Y'], [1, 'N'], ...],
#       static_source : [[0, 'N'], [1, 'N'], ...]
#     }
#   }
# }
def db_load(path):
    db = dict()

    # iterate over apps
    for a in [e for e in path.iterdir() if e.is_dir()]:
        if os.path.basename(a)[0] == ".":
            continue

        app = dict()

        # if there are multiple workloads for this app, take the latest
        bench_wl_folder = None
        suite_wl_folder = None

        changed = False
        for w in [e for e in a.iterdir() if e.is_dir()]:
            if (os.path.basename(w).startswith("benchmark")):
                if bench_wl_folder is None:
                    bench_wl_folder = w
                elif os.path.getctime(w) > os.path.getctime(bench_wl_folder):
                    bench_wl_folder = w
                    changed = True
            elif (os.path.basename(w).startswith("suite")):
                if suite_wl_folder is None:
                    suite_wl_folder = w
                elif os.path.getctime(w) > os.path.getctime(suite_wl_folder):
                    suite_wl_folder = w
                    changed = True
            else:
                error("Detected malformed DB: incorrect name " + str(w) +
                      " (should start with 'benchmark' or 'suite')")
                exit(1)

        if (changed):
            debug("App directory {} has multiple entries".format(str(a)) +
                  ", went for most recent ones: {} and {}".format(str(bench_wl_folder),
                      str(suite_wl_folder)))

        bench_workload = {"dynamic": [], "static_binary": [], "static_source": []}
        found_static_binary = False
        found_static_source = False
        if (bench_wl_folder is not None):
            # if there are multiple runs for this workload, take the latest
            m = None
            i = 0
            for _m in [e for e in bench_wl_folder.iterdir() if e.is_dir()]:
                i += 1
                if m is None or os.path.getctime(_m) > os.path.getctime(m):
                    m = _m

            if (i != 1):
                debug("Workload directory {} has multiple entries".format(
                      str(bench_wl_folder)) + ", went for most recent: {}".format(
                          str(m)))

            # iterate over CSV data files

            ml = list()
            with open(os.path.join(str(m), "data", "dyn.csv"), "r") as f:
                for line in f:
                    if (line[0] == '#'):
                        continue
                    ml.append(re.sub(r"[\n\t\s]*", '', line).split(","))
            bench_workload["dynamic"] = ml

            ml = list()
            try:
                with open(os.path.join(str(m), "data", "static_binary.csv"), "r") as f:
                    for line in f:
                        if (line[0] == '#'):
                            continue
                        ml.append(re.sub(r"[\n\t\s]*", '', line).split(","))
                    found_static_binary = True
            except FileNotFoundError:
                ml = None
            bench_workload["static_binary"] = ml

            ml = list()
            try:
                with open(os.path.join(str(m), "data", "static_sources.csv"), "r") as f:
                    for line in f:
                        if (line[0] == '#'):
                            continue
                        ml.append(re.sub(r"[\n\t\s]*", '', line).split(","))
                    found_static_source = True
            except FileNotFoundError:
                ml = None
            bench_workload["static_source"] = ml
        else:
            debug("App directory %s does not feature benchmark data" % str(a))

        suite_workload = {"dynamic": [], "static_binary": [], "static_source": []}
        if (suite_wl_folder is not None):
            # if there are multiple runs for this workload, take the latest
            m = None
            i = 0
            for _m in [e for e in suite_wl_folder.iterdir() if e.is_dir()]:
                i += 1
                if m is None or os.path.getctime(_m) > os.path.getctime(m):
                    m = _m

            if (i != 1):
                debug("Workload directory {} has multiple entries".format(
                    str(suite_wl_folder)) + ", went for most recent: {}".format(
                        str(m)))

            ml = list()

            # iterate over CSV data files

            ml = list()
            with open(os.path.join(str(m), "data", "dyn.csv"), "r") as f:
                for line in f:
                    if (line[0] == '#'):
                        continue
                    ml.append(re.sub(r"[\n\t\s]*", '', line).split(","))
            suite_workload["dynamic"] = ml

            ml = list()
            try:
                with open(os.path.join(str(m), "data", "static_binary.csv"), "r") as f:
                    for line in f:
                        if (line[0] == '#'):
                            continue
                        ml.append(re.sub(r"[\n\t\s]*", '', line).split(","))
                    found_static_binary = True
            except FileNotFoundError:
                ml = None
            suite_workload["static_binary"] = ml

            ml = list()
            try:
                with open(os.path.join(str(m), "data", "static_sources.csv"), "r") as f:
                    for line in f:
                        if (line[0] == '#'):
                            continue
                        ml.append(re.sub(r"[\n\t\s]*", '', line).split(","))
                    found_static_source = True
            except FileNotFoundError:
                ml = None
            suite_workload["static_source"] = ml
        else:
            debug("App directory %s does not feature test suite data" % str(a))

        if not found_static_binary:
            debug("App directory %s does not feature static binary data" % str(a))
        if not found_static_source:
            debug("App directory %s does not feature static source data" % str(a))

        app["benchmark"] = bench_workload
        app["testsuite"] = suite_workload

        db[os.path.basename(a)] = app

    return db

def get_workloads(db, applist, bench=False, suite=False,
        static_binary=False, static_source=False):

    if "*" in applist and len(applist) > 1:
        warning("* in the application list but other entries are specified: " + str(applist))
        warning("Ignoring them.")
        applist = db.keys()
    elif "*" in applist:
        applist = db.keys()

    workloads = []
    considered_apps = []
    for app in applist:
        tmp = []

        if bench and len(db[app]["benchmark"]["dynamic"]):
            tmp.append(db[app]["benchmark"]["dynamic"])
        elif bench:
            continue

        if suite and len(db[app]["testsuite"]["dynamic"]):
            tmp.append(db[app]["testsuite"]["dynamic"])
        elif suite:
            continue

        if static_binary:
            if (db[app]["benchmark"]["static_binary"] is not None and
               len(db[app]["benchmark"]["static_binary"])):
                tmp.append(db[app]["benchmark"]["static_binary"])
            elif (db[app]["testsuite"]["static_binary"] is not None and
               len(db[app]["testsuite"]["static_binary"])):
                tmp.append(db[app]["testsuite"]["static_binary"])
            else:
                continue

        if static_source:
            if (db[app]["benchmark"]["static_source"] is not None and
               len(db[app]["benchmark"]["static_source"])):
                tmp.append(db[app]["benchmark"]["static_source"])
            elif( db[app]["testsuite"]["static_source"] is not None and
               len(db[app]["testsuite"]["static_source"])):
                tmp.append(db[app]["testsuite"]["static_source"])
            else:
                continue

        workloads.extend(tmp)
        considered_apps.append(app)

    return (workloads, considered_apps)

# returns a tuple of two arrays (required, used) in the form of
# (
#   [90,  18, 0,  12, ...],
#   [100, 34, 10, 45, ...]
# )
# with required[syscall number] = percentage of apps that require the syscall
# e.g., required[0] = 90%, required[1] = 18%, etc.
#
# if passed static_binary=True or static_source=True, the second element of
# the Tuple (used) will be None to represent N/A.
def process_cumulative(db, applist, bench=False, suite=False,
        static_binary=False, static_source=False, ignore_fake=False):
    assert(bench or suite or static_binary or static_source)
    assert('*' not in applist)

    cumulative_required = {}
    cumulative_executed = {}

    # prepopulate
    for i in range(0, MAX_SYSCALL + 1):
        cumulative_required[i] = 0
        cumulative_executed[i] = 0

    n = 0
    for app in applist:
        # collect all relevant measurements
        workloads = get_workloads(db, [app], bench, suite, static_binary, static_source)[0]

        if len(workloads):
            n += 1

        for i in range(0, MAX_SYSCALL + 1):
            exe = False
            req = False
            for w in workloads:
                if bench or suite:
                    if w[i][1] == "Y":
                        exe = True
                        if ignore_fake:
                            if w[i][3] == "N" and w[i][4] == "N":
                                req = True
                        else:
                            if w[i][2] == "N" and w[i][3] == "N" and w[i][4] == "N":
                                req = True
                else:
                    if w[i][1] == "Y":
                        req = True
            if exe:
                cumulative_executed[i] += 1
            if req:
                cumulative_required[i] += 1

    if n:
        # convert to percentages
        for i in range(0, MAX_SYSCALL + 1):
            cumulative_required[i] /= n
            cumulative_required[i] *= 100
            cumulative_executed[i] /= n
            cumulative_executed[i] *= 100

    if static_binary or static_source:
        cumulative_executed = None

    return (cumulative_required, cumulative_executed)

# return, for a given application list, the system calls that are required, those
# that can be faked, stubbed, or faked AND stubbed:
# {
#   required : {
#     1, 4, 44, 221 // system call numbers
#   }
#   stubbed: {...}
#   faked: {...}
#   both: {...}
# }
# FIXME has a lot of duplicated code
def used_by_apps(db, applist, bench=False, suite=False,
        static_binary=False, static_source=False):
    assert(bench or suite or static_binary or static_source)

    if "*" in applist:
        applist = db.keys()

    def _used_by_app(app):
        if app not in db.keys():
            error("%s not in the database." % app)
            error("Valid entries are: " + str(db.keys()))
            exit(1)

        req  = []
        stub = []
        fake = []
        both = []

        # collect all relevant measurements
        workloads = get_workloads(db, [app], bench, suite, static_binary, static_source)[0]

        # populate req, stub, fake
        for i in range(0, MAX_SYSCALL + 1):
            isr = None
            iss = None
            isf = None
            isb = None

            for w in workloads:
                if bench or suite:
                    if (w[i][1] == "N"):
                        continue
                    elif ((w[i][2] == "Y" and w[i][3] == "Y") or
                          (w[i][4] == "Y")):
                        # can be stubbed and faked
                        if (iss is not None or isf is not None):
                            continue
                        isr = False
                        iss = True
                        isf = True
                        isb = True
                    elif (w[i][2] == "Y" and w[i][3] == "N"):
                        # can be stubbed
                        isf = False
                        isb = False
                        if (iss is not None):
                            continue
                        isr = False
                        iss = True
                    elif (w[i][2] == "N" and w[i][3] == "Y"):
                        # can be faked
                        iss = False
                        isb = False
                        if (isf is not None):
                            continue
                        isr = False
                        isf = True
                    elif (w[i][2] == "N" and w[i][3] == "N" and w[i][4] == "N"):
                        # required
                        isr = True
                        iss = False
                        isf = False
                        isb = False
                        break
                else:
                    if (w[i][1] == "Y"):
                        isr = True
                    elif (w[i][1] == "N" and isr is None):
                        isr = False

            if isr:
                req.append(i)
            elif isb:
                both.append(i)
            elif iss:
                stub.append(i)
            elif isf:
                fake.append(i)

        return {"required": req, "stubbed": stub, "faked": fake, "both": both}

    pdb = dict()
    for a in applist:
        pdb[a] = _used_by_app(a)

    req  = []
    stub = []
    fake = []
    both = []

    for i in range(0, MAX_SYSCALL + 1):
        isr = None
        iss = None
        isf = None
        isb = None

        for a in applist:
            if (i in pdb[a]["both"]):
                if (iss is not None or isf is not None):
                    continue
                isr = False
                iss = True
                isf = True
                isb = True
            elif (i in pdb[a]["stubbed"]):
                # can be stubbed
                isf = False
                isb = False
                if (iss is not None):
                    continue
                isr = False
                iss = True
            elif (i in pdb[a]["faked"]):
                # can be faked
                iss = False
                isb = False
                if (isf is not None):
                    continue
                isr = False
                isf = True
            elif (i in pdb[a]["required"]):
                # required
                isr = True
                iss = False
                isf = False
                isb = False
                break

        if isr:
            req.append(i)
        elif isb:
            both.append(i)
        elif iss:
            stub.append(i)
        elif isf:
            fake.append(i)

    return {"required": req, "stubbed": stub, "faked": fake, "both": both}

def container_exists(name):
    runcmd = ["docker", "images"]
    out = subprocess.check_output(runcmd).decode('utf-8')
    return re.search("^%s" % name, out, re.MULTILINE) is not None

def remove_container(name):
    info("Removing stale container...")
    runcmd = ["docker", "container", "rm", name]
    process = subprocess.Popen(runcmd)
    process.wait()

def select_files_for_copy_in_db(dockerfile):
    tmp = dockerfile.split('\n')
    lines = []
    for line in tmp:
        if(line.find("COPY") == 0  or line.find("ADD") == 0):
            aux = line.split(' ')
            for word in aux:
                if(word.find("dockerfile_data") == 0):
                    lines.append(word.split("dockerfile_data/")[1])
                    break
    return lines

def run_tests(path_db, application, workload, path_dockerfile, only_build_docker=False):
    start = datetime.now()

    info("Building container...")
    if not path_dockerfile.exists():
        error("Dockerfile %s does not exist" % str(path_dockerfile))
        return False

    containername = "%s-loupe" % application

    # build container in /tmp to at least detect any reference to a local directory
    # that could be needed
    tmpbuild = get_temp_dir()
    shutil.copyfile(path_dockerfile, os.path.join(tmpbuild, "Dockerfile.%s" % application))
    if (pathlib.Path("./dockerfile_data").exists()):
        shutil.copytree("./dockerfile_data", os.path.join(tmpbuild, "dockerfile_data"))

    cwd = os.getcwd()
    os.chdir(tmpbuild)
    runcmd = ["docker", "build", "--tag", containername, "-f", str(path_dockerfile), "."]
    process = subprocess.Popen(runcmd)

    process.wait()
    os.chdir(cwd)

    ret = process.returncode
    if (ret != 0):
        error("Problem building the container? Error code %d" % ret)
        return False

    if (not container_exists(containername)):
        error("Problem building the container?")
        return False

    if (only_build_docker):
        info("Done building the container, exiting (called with " + ONLY_DOCKER_OPT + ").")
        return True

    info("Running dynamic analysis in the container ({} replicas)...".format(
        NUMBER_GENERATE_REPLICAS))

    def _run_test(n, i, r, s):
        # share build temporary directory to store quiet output
        runcmd = ["docker", "container", "run", "--rm", "--privileged", "-v",
                  tmpbuild + ":" + DOCKER_SHAREDIR, n]
        try:
            local = threading.local()
            local.o = subprocess.check_output(runcmd).decode('utf-8')
            local.split = local.o.split("\n\n")

            r[i] = local.split[0]
            # only write static_results if we are replica 0, they cannot have
            # any variation anyways
            if (i == 0):
                s[0] = local.split[1]
        except subprocess.CalledProcessError as e:
            r[i]  = str(e.output)

    # run configured number of replicas
    threads = [None] * NUMBER_GENERATE_REPLICAS
    results = [None] * NUMBER_GENERATE_REPLICAS
    static_results_array = [None]
    for j in range(NUMBER_GENERATE_REPLICAS // NUMBER_PARALLEL_REPLICAS):
        debug("Starting replica batch (%d/%d), batch size %d" % ((
            j * NUMBER_PARALLEL_REPLICAS) + NUMBER_PARALLEL_REPLICAS,
            NUMBER_GENERATE_REPLICAS, NUMBER_PARALLEL_REPLICAS))

        for i in range(NUMBER_PARALLEL_REPLICAS):
            t = threading.Thread(target=_run_test, args=(containername,
                (j * NUMBER_PARALLEL_REPLICAS) + i, results, static_results_array))
            threads[j + i] = t
            t.start()

        for t in threads:
            if t is not None:
                t.join()

    # sanitize a bit, make sure that the output is sane
    info("Sanitizing replicas outputs...")

    # sanitize binary static output
    if static_results_array[0] is None:
        error("Binary static analysis results are invalid (no results!)")
        return False

    static_results = static_results_array[0].strip()

    if(static_results != "Static analysis skipped"):
        l = 0
        for line in static_results.splitlines():
            if (len(re.sub("[^,]", "", line)) != 1):
                error("Line %d of the binary static analysis output seems " % l +
                    "corrupted. Dumping.")
                print(static_results)
                return False
            l += 1
        if (l != MAX_SYSCALL + 2):
            error("Somehow the binary static analysis output does not have the " +
                "right size... Expected %d lines, got %d. Dumping." % (MAX_SYSCALL + 2, l))
            print(static_results)
            return False

    # sanitize dynamic output
    x = 0
    for r in results:
        if r is None:
            error("Replica %d did not produce result" % x)
            return False

        l = 0
        for line in r.splitlines():
            if (len(re.sub("[^,]", "", line)) != 4):
                error("Line %d of replica %d's test output seems " % (l, x) +
                      "corrupted. Dumping.")
                print(r)
                return False
            l += 1
        if (l != MAX_SYSCALL + 2):
            error("Somehow replica %d's file does not have the right size... " % x +
                  "Expected %d lines, got %d. Dumping." % (MAX_SYSCALL, l))
            print(r)
            return False
        x += 1

    info("Merging replicas outputs...")

    def _merge_cvs(results):
        final_csv = ""
        d = 0
        for i in range(MAX_SYSCALL + 2):
            known = [results[0].splitlines()[i]]
            for r in results:
                if r is not None and r.splitlines()[i] not in known:
                    known.append(r.splitlines()[i])
            if (len(known) != 1):
                sys = known[0].split(",")[0]
                used = "N"
                worksfaked = "Y"
                worksstubbed = "Y"
                worksboth = "Y"

                for k in known:
                    if (len(k.split(",")) < 4):
                        print("Invalid output line: " + k)
                        exit(1)

                    if (k.split(",")[1] == "Y"):
                        used = "Y"
                    if (k.split(",")[2] == "N"):
                        worksfaked = "N"
                        worksboth = "N"
                    if (k.split(",")[3] == "N"):
                        worksstubbed = "N"
                        worksboth = "N"
                final_csv += "{},{},{},{},{}\n".format(sys, used, worksfaked,
                                                       worksstubbed, worksboth)
                d += 1
            else:
                final_csv += known[0] + "\n"
        info("Replicas reported %d differences" % d)
        return final_csv

    def _check_results_valid(csv):
        used = faked = stubbed = both = 0
        for line in csv.splitlines():
            tmp = line.split(',')
            if(tmp[1] == 'Y'):
                used += 1
            if(tmp[2] == 'Y'):
                faked += 1
            if(tmp[3] == 'Y'):
                stubbed += 1
            if(tmp[4] == 'Y'):
                both += 1
        sum = faked + stubbed + both
        if(sum == 0 or sum == used):
            return False
        return True

    out = _merge_cvs(results)
    if(_check_results_valid(out) == False):
        error("Dynamic analysis results are invalid (all or none of the " +
            "syscalls invoked can be stubbed/faked")
        return False

    # write to the db
    info("Writing to the database...")

    hashdf = get_file_hash(path_dockerfile)

    runpath = os.path.join(path_db, application, workload, hashdf)
    if not os.path.exists(runpath):
        os.makedirs(runpath)
        shutil.copyfile(path_dockerfile,
                        os.path.join(runpath, "Dockerfile.%s" % application))
        if (pathlib.Path("./dockerfile_data").exists()):
            os.makedirs(os.path.join(runpath, "dockerfile_data"))
            df = open(path_dockerfile, 'r')
            files = select_files_for_copy_in_db(df.read())
            for file in files:
                source = os.path.abspath(os.path.join("./dockerfile_data/%s" % file))
                destination = os.path.abspath(os.path.join(runpath, "dockerfile_data/%s" % file))
                if os.path.isdir(source):
                    copy_tree(source, destination)
                else:
                    shutil.copyfile(source, destination)
    else:
        info("This run already exists, replacing (see changes with git diff).")
        shutil.rmtree(os.path.join(runpath, "data"))
        if os.path.exists(os.path.join(runpath, "cmd.txt")):
            os.remove(os.path.join(runpath, "cmd.txt"))
        if os.path.exists(os.path.join(runpath, "explore.logs")):
            os.remove(os.path.join(runpath, "explore.logs"))

    if not os.path.exists(os.path.join(runpath, "data")):
        os.makedirs(os.path.join(runpath, "data"))

    with open(os.path.join(runpath, "data", "dyn.csv"), "a+") as outf:
        outf.write(out)

    if(static_results != "Static analysis skipped"):
        with open(os.path.join(runpath, "data", "static_binary.csv"), "a+") as outf:
            outf.write(static_results)

    debug("Outputing additional reproducibility and debugging information...")

    with open(os.path.join(runpath, "cmd.txt"), "a+") as outf:
        outf.write(" ".join(sys.argv))

    # concatenate all explore logs and save them in explore.logs in the database
    repnum = 0
    reg = re.compile('.*\.log')
    for subdir, dirs, files in os.walk(tmpbuild):
        with open(os.path.join(runpath, "explore.logs"), "a+") as outf:
            for file in files:
                if reg.match(file):
                    outf.write("**** Logs of replica #" + str(repnum) + " ****\n")
                    with open(os.path.join(tmpbuild, file), "r") as logf:
                        outf.write(logf.read())
                    repnum += 1

    debug("Checking explorer logs...")
    with open(os.path.join(runpath, "explore.logs")) as logf:
        content = logf.read()
        err = False

        if "Final check analysis for faking failed; the entire set cannot" in content:
            warning("At least one replica reported issues when consolidating stubbing")
            warning("and faking measurements. Please check %s." %
                    os.path.join(runpath, "explore.logs"))
            err = True

        if "[E] " in content:
            warning("At least one replica reported an error. Please check %s."
                    % os.path.join(runpath, "explore.logs"))
            err = True

        if not err:
            debug("explorer logs all OK.")

    end = datetime.now()

    info("Done! Full analysis last %s." % str(end - start))
    print(" -- Make sure to commit the changes to the database :-)")

    return True

def support_plan(applist, supported):
    if "*" in applist:
        applist = db.keys()

    print("Step by step support plan for: " + str(applist))

    per_app_syscalls = {}
    for app in applist:
        per_app_syscalls[app] = {}
        per_app_syscalls[app]["required"] = \
                format_syscall_list(used_by_apps(db, [app], benchmark,
                    testsuite)["required"])
        per_app_syscalls[app]["faked"] = \
                format_syscall_list(used_by_apps(db, [app], benchmark,
                    testsuite)["faked"])
        per_app_syscalls[app]["stubbed"] = \
                format_syscall_list(used_by_apps(db, [app], benchmark,
                    testsuite)["stubbed"])
        per_app_syscalls[app]["both"] = \
                format_syscall_list(used_by_apps(db, [app], benchmark,
                    testsuite)["both"])

    already_supported = []
    for app in per_app_syscalls:
        if set(per_app_syscalls[app]["required"]).issubset(supported):
            already_supported.append(app)
    if already_supported:
        print("- Supported without changes: ", end="")
        print(already_supported)
        for app in already_supported:
            per_app_syscalls.pop(app)

    step = 1
    already_stubbed = set()
    already_faked = set()
    apps_to_support = []
    while per_app_syscalls:
        next_app = list(per_app_syscalls.keys())[0]
        next_app_impl = set(per_app_syscalls[next_app]["required"])\
                .difference(set(supported))
        for app in per_app_syscalls:
            impl_required = set(per_app_syscalls[app]["required"])\
                    .difference(set(supported))
            if len(impl_required) < len(next_app_impl):
                next_app = app
                next_app_impl = impl_required

        stub_needed = (set(per_app_syscalls[next_app]["stubbed"])\
                .union(set(per_app_syscalls[next_app]["both"]))).difference(set(supported))\
                .difference(already_stubbed)
        fake_needed = set(per_app_syscalls[next_app]["faked"]).difference(set(supported)).\
                difference(already_faked)
        already_stubbed = already_stubbed.union(stub_needed)
        already_faked = already_faked.union(fake_needed)

        apps_to_support.append(next_app)
        if next_app_impl or stub_needed or fake_needed:
            print("- Step " + str(step) + " - to support " + ', '.join(apps_to_support) + ":")

        if next_app_impl:
            print("  - implement " + str(format_syscall_list_to_names(next_app_impl)))
        if stub_needed:
            print("  - stub " + str(format_syscall_list_to_names(stub_needed)))
        if fake_needed:
            print("  - fake " + str(format_syscall_list_to_names(fake_needed)))
        supported = supported.union(next_app_impl)
        del per_app_syscalls[next_app]

        # Increment step only if something (implementation/stub/fake) was required
        if next_app_impl or stub_needed or fake_needed:
            apps_to_support = []
            step += 1


# parse arguments and launch the right option
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest='cmd')
parser.add_argument("-v", "--verbose", action="store_true", dest="verbose",
        help="enable debug output")
parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
        help="disable any non-error output")
parser.add_argument("--allow-dirty-db", action="store_true", dest="dirtydb",
        default=False, help="allow dirty DB with uncommited changes")

run_parser = subparsers.add_parser("generate",
        help="run system call usage analysis for an application")

run_parse_req_args = run_parser.add_argument_group('required arguments')
run_parse_req_args.add_argument("-db", "--database", dest="dbpath",
        type=pathlib.Path, required=True, help="path to the database")
run_parse_req_args.add_argument("-a", "--application-name", type=str, required=True,
        help="name of the application to be analyzed (e.g., nginx)", dest="application")
run_parse_req_args.add_argument("-w", "--workload-name", type=str, required=True,
        help="name of the workload (e.g., wrk)", dest="workload")
run_parse_req_args.add_argument("-d", "--dockerfile", type=pathlib.Path, required=True,
        help="path to the dockerfile that performs the analysis")

run_parse_classifier_args = run_parser.add_argument_group('classifier arguments (exactly one required)')
run_parse_classifier_args.add_argument("-b", action="store_true", dest="isbenchmark",
        help="consider this workload as a benchmark")
run_parse_classifier_args.add_argument("-s", action="store_true", dest="issuite",
        help="consider this workload as a testsuite")

run_parse_other_args = run_parser.add_argument_group('optional arguments')
run_parse_other_args.add_argument(ONLY_DOCKER_OPT, action="store_true", dest="onlydocker",
        help="only build the Docker container, do not run the analysis")

search_parser = subparsers.add_parser("search",
        help="retrieve and analyze data from the database")

required_args = search_parser.add_argument_group('required arguments')
required_args.add_argument("-db", "--database", dest="dbpath",
        type=pathlib.Path, required=True, help="path to the database")
required_args.add_argument("-a", "--applications", dest="applist", type=str,
        help="comma-separated list of apps to consider, e.g., 'redis,nginx', '*' for all")
required_args.add_argument("-w", "--workloads", dest="wllist", type=str,
        help="comma-separated list of workloads to consider, e.g., 'bench,suite', '*' for all")

action_args = search_parser.add_argument_group('action arguments')
action_args.add_argument("--show-usage", dest="showusage", action="store_true",
        help="output a list of required/stubbed/faked system calls for this set")
action_args.add_argument("--guide-support", dest="supportfile", type=pathlib.Path,
        help="given the path to a newline separated file of supported system calls, " +
        "output the remaining system calls to implement to support this set")
action_args.add_argument("--cumulative-plot", action="store_true", dest="cumulativeplot",
        help="output a cumulative support plot for this set")
action_args.add_argument("--heatmap-plot", action="store_true",
        help="output a heatmap support plot for this set", dest="heatmapplot")
action_args.add_argument("--paper-histogram-plot", action="store_true",
        help="output the histogram of the paper, ignores passed set", dest="paperhistogramplot")
action_args.add_argument("--export-sqlite", action="store_true",
        help="export the DB as SQLite database")

opt_args = search_parser.add_argument_group('optional arguments')
opt_args.add_argument("--static-source", action="store_true",
        help="also include static source analysis data", dest="ssource")
opt_args.add_argument("--output-sys-names", action="store_true", dest="outputnames",
        help="output system call names instead of numbers")

args = parser.parse_args()

common.ENABLE_VERBOSE = (args.verbose is True)
common.ENABLE_QUIET = (args.quiet is True)
ENABLE_DIRTY_DB = (args.dirtydb is True)

if (args.cmd is None):
    parser.print_help()
    exit(1)

info("Checking database...")
if not db_check(args.dbpath):
    error("Problem with the database, exiting.")
    exit(1)

if (args.cmd == "search"):
    if ((args.applist is None or args.wllist is None) and
            args.paperhistogramplot is not True):
        error("Application list (-a/--applications) and workload list " +
              "(-w/--workloads) are required for this option.")
        error("Call with --help for more information.")
        exit(1)

    common.OUTPUT_NAMES = (args.outputnames is True)

    db = db_load(args.dbpath)

    benchmark = False
    testsuite = False

    if (args.showusage is True or args.cumulativeplot is True or
            args.heatmapplot is True or args.supportfile is not None):
        if "*" in args.wllist and len(args.wllist) > 1:
            warning("* in the workload list but other entries are specified: " +
                    str(args.wllist))
            warning("Ignoring them.")
            benchmark = True
            testsuite = True
        elif "*" in args.wllist:
            benchmark = True
            testsuite = True
        elif "benchmark" in args.wllist or "bench" in args.wllist:
            benchmark = True
        elif "testsuite" in args.wllist or "suite" in args.wllist:
            testsuite = True
        else:
            error("Invalid workload passed (valid: '*', 'benchmark'/'bench', 'testsuite'/'suite')")
            exit(1)

    usage = []

    if (args.showusage is True or args.supportfile is not None):
        usage = used_by_apps(db, args.applist.split(","), bench=benchmark, suite=testsuite)

    if (args.heatmapplot is True or args.cumulativeplot is True or
                                    args.paperhistogramplot is True):
        # build plot container
        info("Building plot container")
        runcmd = ["docker", "build", "--tag", "loupe-plot",
                  "-f", "docker/Dockerfile.loupe-plot", "."]
        process = subprocess.Popen(runcmd)
        process.wait()

        # remove data file
        fileList = glob.glob('./*.dat')
        for filePath in fileList:
            try:
                os.remove(filePath)
            except:
                pass

    if (args.showusage is True):
        print("Required:")
        print(format_syscall_list(usage["required"]))
        print("Can be stubbed:")
        print(format_syscall_list(usage["stubbed"]))
        print("Can be faked:")
        print(format_syscall_list(usage["faked"]))
        print("Can be both stubbed or faked:")
        print(format_syscall_list(usage["both"]))
    elif (args.paperhistogramplot is True):
        apps = ["haproxy", "lighttpd", "memcached", "nginx", "redis",
                "sqlite", "weborf"]

        # process data
        info("Processing data")

        # 1. check that apps are featured in the database
        error = False
        for a in apps:
            if (a not in db.keys()):
                error("Application missing from the database " +
                      "(and required for the plot): " + a)
                error = True

        if (error):
            exit(1)

        # 2. generate data files (haproxy.dat, lighttpd.dat, memcached.dat, nginx.dat,
        # redis.dat, sqlite.dat, webfsd.dat, and weborf.dat)
        missingBars = False
        for a in apps:
            if ("benchmark" not in db[a].keys() or
                len(db[a]["benchmark"].keys()) == 0 or
                len(db[a]["benchmark"]["dynamic"]) == 0):
                missingBars = True
            dat_bench = used_by_apps(db, [a], True, False, False, False)

            if ("testsuite" not in db[a].keys() or
                len(db[a]["testsuite"].keys()) == 0 or
                len(db[a]["testsuite"]["dynamic"]) == 0):
                missingBars = True
            dat_suite = used_by_apps(db, [a], False, True, False, False)

            dat_static_binary = used_by_apps(db, [a], False, False, True, False)
            if (not len(dat_static_binary["required"])):
                missingBars = True

            dat_static_source = used_by_apps(db, [a], False, False, False, True)
            if (not len(dat_static_source["required"])):
                missingBars = True

            with open(os.path.join("./" + a + ".dat"), "a") as outf:
                # header
                outf.write(a + " staticsrc   staticbin   required    stubonly    fakeonly    fakeorstub\n")
                # binary
                outf.write("all     0           {}         0           0           0           0\n".format(
                    str(len(dat_static_binary["required"]))))
                # source
                outf.write("all     {}          0          0           0           0           0\n".format(
                    str(len(dat_static_source["required"]))))
                # suite
                outf.write("suite   0           0          {}          {}          {}          {}\n".format(
                    str(len(dat_suite["required"])), str(len(dat_suite["stubbed"])),
                    str(len(dat_suite["faked"])), str(len(dat_suite["both"]))))
                # bench
                outf.write("bench   0           0          {}          {}          {}          {}\n".format(
                    str(len(dat_bench["required"])), str(len(dat_bench["stubbed"])),
                    str(len(dat_bench["faked"])), str(len(dat_bench["both"]))))

        info("Building plot")
        cwd = os.getcwd()
        runcmd = ["docker", "run", "-it", "--rm", "-v", cwd+ ":/mnt", "loupe-plot",
                  "gnuplot", "/mnt/resources/paper-histogram-plot.gnu"]
        print(" ".join(runcmd))
        process = subprocess.Popen(runcmd)
        process.wait()

        if missingBars:
            warning("The plot is missing bars because certain measurements are not provided")

        # notify user
        print("Plot: ./paper-histogram.svg")
    elif (args.heatmapplot is True):
        # process data
        info("Processing data")

        # we need to be able to use the same dataset for all plots
        considered = get_workloads(db, args.applist.split(","), benchmark,
                testsuite, True, True)[1]
        info("Apps considered in this plot: " + ' '.join(considered))

        cumulative = process_cumulative(db, considered,
                bench=benchmark, suite=testsuite)

        cumulative_binary = process_cumulative(db, considered,
                False, False, True, False)

        cumulative_source = process_cumulative(db, considered,
                False, False, False, True)

        # generate data files (./data-staticsource.dat, ./data-staticbinary.dat,
        # ./data-dynused.dat, ./data-dynstubfake.dat)
        staticsourcedat = ""
        staticbindat = ""
        dynuseddat = ""
        dynstubfakedat = ""

        i = 1
        for sysnumber in reversed(range(MAX_SYSCALL + 2)):
            x = (i - 1) % 24
            y = (i - 1) // 24
            i += 1

            if (x == 0 and y != 0):
                staticsourcedat += "\n"
                staticbindat += "\n"
                dynuseddat += "\n"
                dynstubfakedat += "\n"

            p = 0
            if (sysnumber <= MAX_SYSCALL):
                p = cumulative[1][sysnumber]
            dynuseddat += str(x) + " " + str(y) + " " + str(p) + " " + str(sysnumber) + "\n"

            if (sysnumber <= MAX_SYSCALL):
                p = cumulative[0][sysnumber]
            dynstubfakedat += str(x) + " " + str(y) + " " + str(p) + " " + \
                              str(sysnumber) + "\n"

            if (sysnumber <= MAX_SYSCALL):
                p = cumulative_binary[0][sysnumber]
            staticbindat += str(x) + " " + str(y) + " " + str(p) + " " + \
                              str(sysnumber) + "\n"

            if (sysnumber <= MAX_SYSCALL):
                p = cumulative_source[0][sysnumber]
            staticsourcedat += str(x) + " " + str(y) + " " + str(p) + " " + \
                              str(sysnumber) + "\n"

        with open(os.path.join("./data-staticsource.dat"), "a") as outf:
            outf.write(staticsourcedat)
        with open(os.path.join("./data-staticbinary.dat"), "a") as outf:
            outf.write(staticbindat)
        with open(os.path.join("./data-dynused.dat"), "a") as outf:
            outf.write(dynuseddat)
        with open(os.path.join("./data-dynstubfake.dat"), "a") as outf:
            outf.write(dynstubfakedat)

        # generate dumps (only used separately for correlation checks)
        dynuseddat_dump = ""
        dynstubfakedat_dump = ""
        dynstubdat_dump = ""

        cumulative_nofake = process_cumulative(db, considered,
                bench=benchmark, suite=testsuite, ignore_fake=True)

        for sysnumber in range(MAX_SYSCALL):
            p = cumulative[1][sysnumber]
            dynuseddat_dump += str(sysnumber) + "\t" + str(p) + "\n"

            p = cumulative[0][sysnumber]
            dynstubfakedat_dump += str(sysnumber) + "\t" + str(100 - p) + "\n"

            p = cumulative_nofake[0][sysnumber]
            dynstubdat_dump += str(sysnumber) + "\t" + str(100 - p) + "\n"

        with open(os.path.join("./data-dynused-dump.dat"), "a") as outf:
            outf.write(dynuseddat_dump)
        with open(os.path.join("./data-dynstubfake-dump.dat"), "a") as outf:
            outf.write(dynstubfakedat_dump)
        with open(os.path.join("./data-dynstub-dump.dat"), "a") as outf:
            outf.write(dynstubdat_dump)

        info("Building plot")
        cwd = os.getcwd()
        runcmd = ["docker", "run", "-it", "--rm", "-v", cwd+ ":/mnt", "loupe-plot",
                  "gnuplot", "/mnt/resources/heatmap-plot.gnu"]
        print(" ".join(runcmd))
        process = subprocess.Popen(runcmd)
        process.wait()

        # notify user
        print("Plot: ./heapmap-dynamic-used.svg")
        print("Plot: ./heapmap-dynamic-stubfake.svg")
        print("Plot: ./heapmap-static-binary.svg")

        if (args.ssource is not True):
            os.remove("./heapmap-static-source.svg")
        else:
            print("Plot: ./heapmap-static-source.svg")
    elif (args.cumulativeplot is True):
        # process data
        info("Processing data")

        # we need to be able to use the same dataset for all lines
        considered = get_workloads(db, args.applist.split(","), benchmark,
                False, False, False)[1]
        info("Apps considered in this plot: " + ' '.join(considered))

        cumulative = process_cumulative(db, considered, bench=benchmark,
                suite=testsuite)

        cumulative_binary = process_cumulative(db, considered,
                False, False, True, False)

        cumulative_source = process_cumulative(db, considered,
                False, False, False, True)

        # generate data.dat
        d = {}
        c = 0
        for i in sorted(cumulative_binary[0].values(), reverse=True):
            d[c] = i;
            c+=1

        bin = "# x\tstatic-binary\n" + "\n".join(["%s\t%s" % (k,v) for (k,v) in (d.items())])

        d = {}
        c = 0
        for i in sorted(cumulative_source[0].values(), reverse=True):
            d[c] = i;
            c+=1

        src = "# x\tstatic-source\n" + "\n".join(["%s\t%s" % (k,v) for (k,v) in (d.items())])

        d = {}
        c = 0
        for i in sorted(cumulative[0].values(), reverse=True):
            d[c] = i;
            c+=1

        req = "# x\tdyn-req\n" + "\n".join(["%s\t%s" % (k,v) for (k,v) in (d.items())])

        d = {}
        c = 0
        for i in sorted(cumulative[1].values(), reverse=True):
            d[c] = i;
            c+=1

        exe = "# x\tdyn-exe\n" + "\n".join(["%s\t%s" % (k,v) for (k,v) in (d.items())])

        with open(os.path.join("./data.dat"), "a") as outf:
            outf.write(bin)
            outf.write("\n\n\n")
            outf.write(src)
            outf.write("\n\n\n")
            outf.write(exe)
            outf.write("\n\n\n")
            outf.write(req)

        info("Building plot")
        cwd = os.getcwd()
        runcmd = ["docker", "run", "-it", "--rm", "-v", cwd+ ":/mnt", "loupe-plot",
                  "gnuplot", "/mnt/resources/cumulative-plot.gnu"]
        print(" ".join(runcmd))
        process = subprocess.Popen(runcmd)
        process.wait()

        # notify user
        print("Plot: ./cumulative-nostatic.svg")
        print("Plot: ./cumulative-nosource.svg")

        if (args.ssource is True):
            print("Plot: ./cumulative-nobinary.svg")
            print("Plot: ./cumulative-all.svg")
        else:
            os.remove("./cumulative-nobinary.svg")
            os.remove("./cumulative-all.svg")
    elif args.supportfile is not None:
        supported = set(format_syscall_list(open_syscall_file(args.supportfile)))
        required  = set(format_syscall_list(usage["required"]))
        stubbed   = set(format_syscall_list(usage["stubbed"]))
        faked     = set(format_syscall_list(usage["faked"]))
        print("Missing a full implementation:")
        print(str(format_syscall_list_to_names(list(required.difference(supported)))))
        print("Missing a stub:")
        print(str(format_syscall_list_to_names(list(stubbed.difference(supported)))))
        print("Missing a fake:")
        print(str(format_syscall_list_to_names(list(faked.difference(supported)))))

        support_plan(args.applist.split(","), supported)

    else:
        warning("Not implemented yet.")
        exit(0)

if (args.cmd == "generate"):
    wl = args.workload
    if (args.isbenchmark is True) and (args.issuite is True):
        error("Workload cannot be both a benchmark (-b) and a test suite (-s)")
        exit(1)
    elif (args.isbenchmark is True):
        wl = "benchmark-" + wl
    elif (args.issuite is True):
        if wl != "":
            wl = "suite-" + wl
        else:
            wl = "suite"
    else:
        ans = str(input("Is this workload a benchmark or a testsuite? [bench/suite] "))
        if ans.lower() == "b" or ans.lower() == "benchmark" or ans.lower() == "bench":
            wl = "benchmark-" + wl
        elif ans.lower() == "s" or ans.lower() == "testsuite" or ans.lower() == "suite":
            wl = "suite-" + wl
        else:
            error("Cannot understand that answer ('%s')." % ans)
            exit(1)

    # check Dockerfile for obvious errors
    # we just provide that as a quick sanity check for confused users
    # hopefully this does reduce the amount of bug reports we get :)
    debug("Checking Dockerfile...")
    with open(args.dockerfile) as dockerf:
        content = dockerf.read()
        if "loupe-base:latest" not in content:
            warning("Dockerfile (%s) does not base on the loupe-base container "
                    "- here be dragons." % args.dockerfile)

        if "explore.py" not in content:
            error("Dockerfile (%s) does not seem to be calling explore.py in "
                  "the CMD rule." % args.dockerfile)
            exit(1)

        if "--output-csv" not in content:
            error("Dockerfile (%s) calls explore.py without the --output-csv "
                  "option, which is mandatory when using the loupe wrapper." % args.dockerfile)
            exit(1)

        if "--final-check" not in content:
            warning("Dockerfile (%s) calls explore.py without the --final-check "
                  "option, which is recommended." % args.dockerfile)

    # remove potential logs from previous runs
    try:
       os.remove(QUIET_LOG)
    except OSError:
        pass

    ex = run_tests(args.dbpath, args.application, wl, args.dockerfile, args.onlydocker)

    if not ex:
        exit(1)
    exit(0)
