#!/usr/bin/env python3

"""
Drive ctest testing of scream for a complete set of tests. This will be our
gold standard to determine if the code is working or not on the current platform.
For batch machines, this script expects to already be on a compute node; it does not
do batch submissions.

IMPORTANT: the default behavior of this script *changes your environment*,
           by loading machine-specific modules and setting machine-specific
           env vars. To prevent this behavior, use --preserve-env flag.

Baselines: By default, test-all-scream will not run baseline tests. If you set
-b AUTO, baseline tests will be done with the pre-existing public baselines in
the location specified by the machine spec. You can regenerate
baselines any time by using the -g flag, but be aware this will impact everyone
if you regenerate the public baselines. You can change the target baseline area
using -b $path. If -g is provided, no tests will be run; -g means generate only.

The general workflow for baseline-changing PRs is:
1) Issue PR
2) AT will fail with differences in the baseline tests
3) Review and merge.
4) Ask JimF or LucaB for a bless to baselines on mappy and weaver.

If you are developing a branch and baselines tests are failing unexpectedly, it is
likely that your branch has fallen out of date. You should upstream merge or rebase
your branch.

To sum up, the baseline handling for test-all-scream should basically match what we
do for create_test tests, the only difference is that test-all-scream does baseline
comparison tests by default.
"""

from utils import check_minimum_python_version
check_minimum_python_version(3, 4)

import argparse, sys, pathlib

from test_factory import get_test_name_dict
from test_all_scream import TestAllScream

###############################################################################
def parse_command_line(args, description):
###############################################################################
    parser = argparse.ArgumentParser(
        usage="""\n{0} <ARGS> [--verbose]
OR
{0} --help

\033[1mEXAMPLES (assumes user is on machine mappy):\033[0m
    \033[1;32m# Run all tests on current machine using the SCREAM-approved env for this machine \033[0m
    > cd $scream_repo/components/eamxx
    > ./scripts/{0} -m mappy

    \033[1;32m# Run all tests on current machine with defaut behavior except using your current shell env \033[0m
    > cd $scream_repo/components/eamxx
    > ./scripts/{0} --preserve-env -m mappy

    \033[1;32m# Run all tests on current machine with default behavior except using non-default baselines \033[0m
    > cd $scream_repo/components/eamxx
    > ./scripts/{0} -m mappy --baseline-dir=PATH_TO_BASELINES

    \033[1;32m# Run all tests on current machine with default behavior except using local baselines \033[0m
    > cd $scream_repo/components/eamxx
    > ./scripts/{0} -m mappy --baseline-dir=LOCAL

    \033[1;32m# Run only the dbg test on current machine with default behavior otherwise\033[0m
    > cd $scream_repo/components/eamxx
    > ./scripts/{0} -m mappy -t dbg
""".format(pathlib.Path(args[0]).name),
        description=description,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )

    parser.add_argument("-cxx","--cxx-compiler", help="C++ compiler", default=None)
    parser.add_argument("-f90","--f90-compiler", help="F90 compiler", default=None)
    parser.add_argument("--c-compiler", help="C compiler", default=None)

    parser.add_argument("-s", "--submit", action="store_true", help="Submit results to dashboad")
    parser.add_argument("-p", "--parallel", action="store_true",
                        help="Launch the different build types stacks in parallel")

    parser.add_argument("-g", "--generate", action="store_true",
        help="Instruct test-all-scream to generate baselines from current commit. Skips tests")

    parser.add_argument("-b", "--baseline-dir",
        help="Directory where baselines should be read from (or written to, if -g is used). Default is None which skips all baseline tests. AUTO means use public baselines. You can also use LOCAL to manage baselines in your local work dir. Lastly, you can provide a path here as well.")

    parser.add_argument("-m", "--machine",
        help="Provide machine name. This is *always* required. It can, but does not"
"have to, match SCREAM_MACHINE. You can decorate this with compiler"
"info if a machine supports multiple compiler types. This value will be"
"used as the CTEST_SITE for cdash if the tests are submitted. It is"
"expected that a scream machine file exists for this value.")

    parser.add_argument("--config-only", action="store_true",
            help="In the testing phase, only run config step, skip build and tests")

    parser.add_argument("-c", "--custom-cmake-opts", action="append", default=[],
        help="Extra custom options to pass to cmake. Can use multiple times for multiple cmake options. The -D is added for you")

    parser.add_argument("-e", "--custom-env-vars", action="append", default=[],
        help="Extra custom environment variables to be used. These will override"
"(if applicable) whatever was found in machine_specs. Each -e flag"
"supports a single env var, so to pass multiple env var, do -e 'KEY1=VALUE1' -e 'KEY2=VALUE2' ")

    parser.add_argument("--preserve-env", action="store_true",
                        help="Whether to skip machine env setup, and preserve the current user env (useful to manually test new modules)")

    choices_doc = ", ".join(["'{}' ({})".format(k, v) for k, v in get_test_name_dict().items()])
    parser.add_argument("-t", "--test", dest="tests", action="append", default=[],
                        help=f"Only run specific test configurations, choices={choices_doc}")

    parser.add_argument("-l", "--local", action="store_true",
                        help="Allow to not specify a machine name, and have test-all-scream to look"
                             "for '~/.cime/scream_mach_specs.py' for machine specifications.")

    parser.add_argument("-r", "--root-dir",
                        help="The root directory of the scream src you want to test. "
                        "Default will be the scream src containing this script.")

    parser.add_argument("-w", "--work-dir",
        help="The work directory where all the building/testing will happen. Defaults to ${root_dir}/ctest-build")
    parser.add_argument("--quick-rerun", action="store_true",
                        help="Do not clean the build dir, and do not reconfigure. Just (incremental) build and test.")

    parser.add_argument("--quick-rerun-failed", action="store_true",
                        help="Do not clean the build dir, and do not reconfigure. Just (incremental) build and retest failed tests only.")

    parser.add_argument("--make-parallel-level", action="store", type=int, default=0,
        help="Max number of jobs to be created during compilation. If not provided, use default for given machine.")

    parser.add_argument("--ctest-parallel-level", action="store", type=int, default=0,
        help="Force to use this value for CTEST_PARALLEL_LEVEL. If not provided, use default for given machine.")

    parser.add_argument("-x", "--extra-verbose", action="store_true",
                        help="Have ctest run in extra-verbose mode, which should print full output from all tests")

    parser.add_argument("--limit-test-regex",
                        help="Limit ctest to running only tests that match this regex")

    parser.add_argument("--test-level", action="store", choices=("at", "nightly", "experimental"), default="at",
                        help="Set the testing level.")

    parser.add_argument("--test-size", action="store", choices=("short", "medium", "long"),
                        help="Set the testing level. Defaults to medium unless the test is cov or mem_check"
                        "(short is default in those cases).")

    return parser.parse_args(args[1:])

###############################################################################
def _main_func(description):
###############################################################################
    tas = TestAllScream(**vars(parse_command_line(sys.argv, description)))

    success = tas.test_all_scream()

    print("OVERALL STATUS: {}".format("PASS" if success else "FAIL"))

    sys.exit(0 if success else 1)

###############################################################################

if (__name__ == "__main__"):
    _main_func(__doc__)
