#!/usr/bin/env python
# __BEGIN_LICENSE__
#  Copyright (c) 2009-2026, United States Government as represented by the
#  Administrator of the National Aeronautics and Space Administration. All
#  rights reserved.
#
#  The NGT platform is licensed under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance with the
#  License. You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
# __END_LICENSE__

import sys, argparse, subprocess, re, os, math, time, glob, shutil, math, platform
import os.path as P

# Set up the path to Python modules about to load
basepath    = os.path.abspath(sys.path[0])
pythonpath  = os.path.abspath(basepath + '/../Python')  # for dev ASP
libexecpath = os.path.abspath(basepath + '/../libexec') # for packaged ASP
sys.path.insert(0, basepath) # prepend to Python path
sys.path.insert(0, pythonpath)
sys.path.insert(0, libexecpath)

import asp_system_utils, asp_string_utils, asp_cmd_utils
asp_system_utils.verify_python_version_is_supported()
from asp_stereo_utils import * # must be after the path is altered above

# Prepend to system PATH
os.environ["PATH"] = libexecpath + os.pathsep + os.environ["PATH"]

# This is explained below
if 'ASP_LIBRARY_PATH' in os.environ:
    os.environ['LD_LIBRARY_PATH'] = os.environ['ASP_LIBRARY_PATH']

# We will not symlink PC.tif and RD.tif which will be VRT files,
# and neither the log files. We will symlink L.txt and R.txt.
skip_symlink_expr = r'^.*?-(PC\.tif|RD\.tif|log.*?\.txt)$'

def check_system_memory(opt, args, settings):
    '''Issue a warning when doing correlation if our selected options
    are estimated to exceed available RAM. Currently only the ASP SGM
    and MGM have large memory requirements.'''

    if asp_system_utils.run_with_return_code(['which', 'free']) != 0:
        # No 'free' exists. This is the case on OSX.
        return

    try:
        alg = stereo_alg_to_num(settings['stereo_algorithm'][0])
        if alg == VW_CORRELATION_BM or alg >= VW_CORRELATION_OTHER:
            return

        # This is the processor count code, won't work if other
        #  machines have a different processor count.
        num_procs = opt.processes
        if opt.processes is None:
            num_procs = get_num_cpus()

        # Memory usage calculations
        bytes_per_mb        = 1024*1024
        est_bytes_per_pixel = 8 # This is a very rough estimate!

        num_tile_pixels = pow(int(settings['corr_tile_size'][0]), 2)
        baseline_mem    = (num_tile_pixels*est_bytes_per_pixel) / bytes_per_mb
        sgm_ram_limit   = int(settings['corr_memory_limit_mb'][0])
        ram_per_process = baseline_mem + sgm_ram_limit
        est_ram_usage   = int(num_procs * ram_per_process)

        print('Warning: Estimated maximum memory consumption is '
                  + str(est_ram_usage) + ' MB. To lower memory use, '
                  + 'consider reducing the number of processes or lowering '
                  + '--corr-memory-limit-mb.')

    except:
        # Don't let an error here prevent the tool from running.
        print('Warning: Error checking system memory, skipping the memory test!')
        return

def sym_link_prev_run(prev_run_prefix, out_prefix):
    '''Sym link files from a previous run up to triangulation to the
    output directory of this run. We must not symlink directories from
    the previous run as then the new results will be written there.
    Do not sym link the previous point cloud, DEM, etc., also not
    VRT files as those will not be valid.
    '''

    # Ensure we do not wipe the run we want to reuse    
    if prev_run_prefix == out_prefix:
      raise Exception('The previous run prefix is the same as the current run prefix.')
      
    curr_dir = os.path.dirname(out_prefix)
    mkdir_p(curr_dir)
    
    for ext in ['L.tif', 'R.tif', 'L-cropped.tif', 'R-cropped.tif', 'L.tsai', 'R.tsai',
                'L_cropped.tif', 'R_cropped.tif', 
                'L_sub.tif', 'R_sub.tif', 'Mask_sub.tif', 'D_sub.tif', 'D_sub_spread.tif',
                '.vwip', 'L.txt', 'R.txt', '.exr', '.match', 'GoodPixelMap.tif',
                'F.tif', 'stats.tif', 'Mask.tif', 'bathy_mask.tif']:
        files = glob.glob(prev_run_prefix + '*' + ext)
        for f in files:
            if os.path.isdir(f): continue # Skip folders
            dst_f = f[len(prev_run_prefix):] # strip the prefix
            dst_f = out_prefix + dst_f
            rel_src = os.path.relpath(f, curr_dir)
            if os.path.exists(dst_f):
                os.remove(dst_f) # this could be some dangling thing
            os.symlink(rel_src, dst_f)

def create_subdirs_symlink(opt, args, settings):
    """
    Create the list of tiles and the corresponding subdirectories. Create
    symbolic links in each subdirectory to actual files in parent run directory.
    Don't symlink to RD.tif or PC.tif as those will be files which actually need
    to be created in each subdirectory. Return the created subdirectories, and
    save their list to disk.
    """
    
    out_prefix = settings['out_prefix'][0]
    tiles = produce_tiles(opt, args, settings, opt.job_size_w, opt.job_size_h)
    subdirs = readDirList(out_prefix)

    try:
        parentDir = os.path.dirname(out_prefix)
        mkdir_p(parentDir)
    except:
        pass
    
    for tile_id in range(len(subdirs)):

        tile        = tiles[tile_id]    
        subdir      = subdirs[tile_id]
        tile_prefix = subdir + "/" + tile.name_str()
    
        if opt.dryrun:
            print("mkdir -p %s" % subdir)
            print("Soft linking via %s %s" % (tile_prefix, out_prefix))
            continue
            
        mkdir_p(subdir)
        files = glob.glob(out_prefix + '*') # the files in the parent directory
        for f in files:
            if os.path.isdir(f): continue # Skip folders
            rel_src = os.path.relpath(f, subdir)
            m = re.match(skip_symlink_expr, rel_src)
            if m: continue # won't sym link certain patterns
            # Make a symlink from main folder to the tile folder
            dst_f = f.replace(out_prefix, tile_prefix)
            if os.path.lexists(dst_f): continue
            os.symlink(rel_src, dst_f)

    return subdirs

def rename_files(settings, subdirs, postfix_in, postfix_out):
    """
      Rename tile_dir/file_in.tif to tile_dir/file_out.tif.
      Skip symlinks, such as when this directory was processed before.
    """
    for subdir in subdirs:
        tile = dirToTile(subdir)
        prefix = subdir + "/" + tile.name_str()
        filename_in  = prefix + postfix_in
        filename_out = prefix + postfix_out
        if os.path.isfile(filename_in) and (not os.path.islink(filename_in)):
            if os.path.exists(filename_out):
                os.remove(filename_out)
            os.rename(filename_in, filename_out)

def create_symlinks_for_multiview(opt, args, settings):
    """
    Running parallel_stereo for each pair in a multiview run
    creates files like:
      out-prefix-pair2/2-4096_4096_1629_1629/4096_4096_1629_1629-F.tif
    To run parallel_stereo for tri, we expect this at
      out-prefix-4096_4096_1629_1629/4096_4096_1629_1629-pair2/2-F.tif
    Create the latter as a sym link.
    """
    subdirs = create_subdirs_symlink(opt, args, settings) # symlink L.tif, etc
    
    for s in sorted(settings.keys()):
        
        # Find the settings for each stereo command that together comprise the
        # multiview run.
        # TODO(oalexan1): This should be factored out as a function, and the above
        # loop should iterate over the output of this function.
        m = re.match(r'multiview_command', s)
        if not m:
            continue
        local_settings=run_and_parse_output("stereo_parse", settings[s][1:],
                                            sep, opt.verbose)
        local_prefix = local_settings['out_prefix'][0]
        m = re.match(r'^(.*?)-pair(\d+)\/', local_prefix)
        if not m:
            continue
        base_prefix = m.group(1)
        index       = str(m.group(2))
        
        for tile_id in range(len(subdirs)):
            subdir = subdirs[tile_id]
            tile = dirToTile(subdir)
            tile_str = tile.name_str()
            src_pref = tile_dir(base_prefix + '-pair' + index + '/' + index, tile) + '/' + tile_str
            dst_dir  = tile_dir(base_prefix, tile) + '/' + tile_str + '-pair' + index
            mkdir_p(dst_dir)

            dst_pref = dst_dir + '/' + index
            files = glob.glob(src_pref + '*')
            for f in files:
                m = re.match(skip_symlink_expr, f)
                if m:
                    continue # won't sym link certain patterns
                m = re.match(r'^' + src_pref + '(.*?)$', f)
                if not m:
                    continue
                suff    = m.group(1)
                src_f   = src_pref + suff
                dst_f   = dst_pref + suff
                rel_src = os.path.relpath(src_f, dst_dir)
                if os.path.lexists(dst_f):
                    continue
                os.symlink(rel_src, dst_f)


def get_num_nodes(nodes_list):

    if nodes_list is None:
        return 1 # local machine

    # Count the number of nodes without repetition (need this for
    # Pleiades).
    nodes = {}
    num_nodes = 0
    try:
        fh = open(nodes_list, "r")
        for line in fh:
            if re.match(r'^\s*$', line): continue # skip empty lines
            matches = re.match(r'^\s*([^\s]*)', line)
            if matches:
                nodes[matches.group(1)] = 1

        num_nodes = len(nodes)
    except Exception as e:
        die(e)
    if num_nodes == 0:
        raise Exception('The list of computing nodes is empty')

    return num_nodes

def get_best_procs_threads(step, opt, settings):
    # Decide the best number of processes to use on a node, and how
    # many threads to use for each process.  There used to be some
    # fancy logic, see below, but it does not work well. For now,
    # use many processes and one thread per process.

    # We assume all machines have the same number of CPUs (cores)
    num_cpus = get_num_cpus()

    # Respect user's choice for the number of processes
    num_procs = num_cpus
    if opt.processes is not None:
        num_procs = opt.processes

    # Same for the number of threads.
    num_threads = 1
    if opt.threads_multi is not None:
        num_threads = opt.threads_multi
        
    # Old code, now turned off.
    if 0:
        if step == Step.corr:
            tile_size = int(settings['corr_tile_size'][0])
        elif step == Step.rfne:
            tile_size = int(settings['rfne_tile_size'][0])
        elif step == Step.tri:
            tile_size = int(settings['tri_tile_size'][0])
        else:
            raise Exception('Stereo step %d must be executed on a single machine.' \
                            % step)

        # We use the observation that each tile uses one thread,
        # so we need to find how many tiles are in the given job.
        num_threads = int(opt.job_size_w*opt.job_size_h/tile_size/tile_size)
        if num_threads > num_cpus: num_threads = num_cpus
        if num_threads <= 0: num_threads = 1

        # For triangulation, we need to consider the case of ISIS cameras
        # when we must use 1 thread unless CSM sensor models have been provided.
        if step == Step.tri:
            cam_info = " ".join(settings['in_file1'] + settings['in_file2'] + \
                                settings['cam_file1'] + settings['cam_file2'])
            m1 = re.search('\\.cub\\b',  cam_info, re.IGNORECASE)
            m2 = re.search('\\.json\\b', cam_info, re.IGNORECASE)
            m3 = re.search('\\.isd\\b',  cam_info, re.IGNORECASE)
            if m1 and not (m2 or m3):
                num_threads = 1

        num_procs = int(math.ceil(float(num_cpus)/num_threads))

    if opt.verbose:
        print("For stage %d, using %d threads and %d processes." %
              (step, num_threads, num_procs))

    return (num_procs, num_threads)

# Launch GNU Parallel for all tiles, it will take care of distributing
# the jobs across the nodes and load balancing. The way we accomplish
# this is by calling this same script but with --tile-id <num>.
def spawn_to_nodes(step, opt, settings, args, subdirs):

    if opt.processes is None or opt.threads_multi is None:
        # The user did not specify these. We will find the best
        # for their system.
        (procs, threads) = get_best_procs_threads(step, opt, settings)
    else:
        procs = opt.processes
        threads = opt.threads_multi

    asp_cmd_utils.wipe_option(args, '--processes', 1)
    asp_cmd_utils.wipe_option(args, '--threads-multiprocess', 1)
    args.extend(['--processes', str(procs)])
    args.extend(['--threads-multiprocess', str(threads)])

    # Each tile has an index in the list of tiles. There can be a huge amount of
    # tiles, and for that reason we store their indices in a file, rather than
    # putting them on the command line. Keep this file in the run directory.
    out_prefix = settings['out_prefix'][0]
    tiles_index = stereoTilesIndex(out_prefix)
    mkdir_p(os.path.dirname(tiles_index))
    f = open(tiles_index, 'w')
    for i in range(len(subdirs)):
        f.write("%d\n" % i)
    f.close()

    # Reset number of done tiles for this step
    reset = True
    updateNumDoneTiles(out_prefix, stereoProgName(step), reset)
    
    # Use GNU parallel with given number of processes.
    # TODO(oalexan1): Run 'parallel' using the runInGnuParallel() function call,
    # when the ASP_LIBRARY_PATH trick can be fully encapsulated in the
    # asp_system_utils.py code rather than being needed for each tool.
    cmd = ['parallel', '--will-cite', '--env', 'ASP_DEPS_DIR', '--env', 'PATH', '--env', 'LD_LIBRARY_PATH', '--env', 'ASP_LIBRARY_PATH', '--env', 'PYTHONHOME', '--env', 'ISISROOT', '-u', '-P', str(procs), '-a', tiles_index]
    if which(cmd[0]) is None:
        raise Exception('Need GNU Parallel to distribute the jobs.')

    if opt.nodes_list is not None:
        cmd += ['--sshloginfile', opt.nodes_list]
    if opt.ssh is not None:
        cmd += ['--ssh', opt.ssh]
    if opt.parallel_options is not None:
        cmd += opt.parallel_options.split(' ')
        
    # Assemble the options which we want GNU parallel to not mess up
    # with. 
    args_copy = args[:] # deep copy
    args_copy += ["--work-dir", opt.work_dir]
    if opt.isisroot  is not None: 
        args_copy += ["--isisroot",  opt.isisroot]
    if opt.isisdata is not None: 
        args_copy += ["--isisdata", opt.isisdata]
    
    # Quote entities that have spaces in them. Don't quote quantities
    # already quoted.
    for index, arg in enumerate(args_copy):
        if re.search(r"[ \t]", arg) and arg[0] != '\'':
            args_copy[index] = '\'' + arg + '\''
            
    python_path = sys.executable # children must use same Python as parent
    start = step; stop = start + 1
    
    # Put options in a single string.
    args_str = python_path + " "              + \
               " ".join(args_copy)            + \
               " --entry-point " + str(start) + \
               " --stop-point " + str(stop) 
    args_str += " --tile-id {}"
    cmd += [args_str]

    # This is a bugfix for RHEL 8. The 'parallel' program fails to start with ASP's
    # libs, so temporarily hide them.
    if 'LD_LIBRARY_PATH' in os.environ:
        os.environ['ASP_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
        os.environ['LD_LIBRARY_PATH'] = ''

    asp_system_utils.generic_run(cmd, opt.verbose)

    # Undo the above
    if 'ASP_LIBRARY_PATH' in os.environ:
        os.environ['LD_LIBRARY_PATH'] = os.environ['ASP_LIBRARY_PATH']

def tile_run(prog, args, opt, settings, tile, **kw):
    '''Job launch wrapper for a single tile'''

    if prog != 'stereo_blend':  # Set collar_size argument to zero in almost all cases.
        asp_cmd_utils.set_option(args, '--sgm-collar-size', [0])

    # Get tool path and output prefix
    binpath = asp_system_utils.bin_path(prog)
    out_prefix = settings['out_prefix'][0]

    # Measure the memory usage on Linux and elapsed time
    timeCmd = []
    if 'linux' in sys.platform and os.path.exists('/usr/bin/time'):
        timeCmd = ['/usr/bin/time', '-f', prog + \
                   ': elapsed=%E ([hours:]minutes:seconds), memory=%M (kb)']
    
    try:
        # Get tile folder
        tile_dir_prefix = tile_dir(out_prefix, tile) + "/" + tile.name_str()

        # When using SGM correlation, increase the output tile size.
        # - The output image will contain more populated pixels but 
        #   there will be no other change.
        adjusted_tile = grow_crop_tile_maybe(settings, prog, tile)
        if adjusted_tile.width <= 0 or adjusted_tile.height <= 0:
            return # the produced tile is empty
        
        # Increase the processing block size for the tile so we process
        # the entire tile in one go.
        if use_padded_tiles(settings) and prog == 'stereo_corr':
            curr_tile_size = max(adjusted_tile.width, adjusted_tile.height)
            asp_cmd_utils.set_option(args, '--corr-tile-size', [curr_tile_size])

        # Set up the call string
        call = [binpath]
        call.extend(args)

        if opt.threads_multi is not None:
            asp_cmd_utils.wipe_option(call, '--threads', 1)
            call.extend(['--threads', str(opt.threads_multi)])

        cmd = call + ['--trans-crop-win'] + adjusted_tile.as_array() # append the tile

        # This is a bugfix for when the output prefix is the same as the
        # bundle-adjust-prefix. Let the stereo executables parse these as it is
        # better equipped for that.
        cmd += ['--output-prefix-override', tile_dir_prefix]

        if opt.dryrun:
            print(" ".join(cmd))
            return
        if opt.verbose:
            print(" ".join(cmd))

        resetStatus = False # for updating the status of done tiles

        # See if perhaps we can skip correlation
        if prog == 'stereo_corr' and opt.resume_at_corr:

            D = tile_dir_prefix + '-D.tif'
            if (not os.path.islink(D)) and asp_system_utils.is_valid_image(D):
                # The disparity D.tif is valid and not a symlink. No need
                # to recreate it.
                updateNumDoneTiles(out_prefix, prog, resetStatus)
                return

            Dnosym = tile_dir_prefix + '-Dnosym.tif'
            if (not os.path.islink(Dnosym)) and asp_system_utils.is_valid_image(Dnosym):
                # In a previous run D.tif was renamed to Dnosym.tif
                # and D.tif was made into a symlink. Still good.
                # Just undo the rename.
                if os.path.exists(D):
                    os.remove(D)
                os.rename(Dnosym, D)
                updateNumDoneTiles(out_prefix, prog, resetStatus)
                return
            
            # We are left with the situation that there is no image which is both
            # valid and not a symlink. Perhaps D does not exist or is corrupted.
            # Then wipe D and Dnosym, if present, and redo the correlation.
            print("Will run correlation to create a valid image for " + D)
            if os.path.exists(D):
                os.remove(D)
            if os.path.exists(Dnosym):
                os.remove(Dnosym)

        cmd = timeCmd + cmd
        (out, err, status) = asp_system_utils.executeCommand(cmd, realTimeOutput = True)

        if len(timeCmd) > 0:
            print(err)
            usage_file = tile_dir_prefix + "-" + prog + "-resource-usage.txt"
            with open(usage_file, 'w') as f:
                f.write(err)

        if status != 0:
            raise Exception('Stereo step ' + kw['msg'] + ' failed')
        
        if status == 0:
          updateNumDoneTiles(out_prefix, prog, resetStatus)

    except OSError as e:
        raise Exception('%s: %s' % (binpath, e))

def normal_run(prog, opt, args, out_prefix, **kw):
    '''Job launch wrapper for a non-tile stereo call.'''

    binpath = asp_system_utils.bin_path(prog)
    call = [binpath]
    call.extend(args)
    
    if opt.threads_single is not None:
        asp_cmd_utils.wipe_option(call, '--threads', 1)
        call.extend(['--threads', str(opt.threads_single)])
    call = reduce_num_threads(call)

    if opt.dryrun:
        print('%s' % ' '.join(call))
        return
    
    if prog != "stereo_tri":
        # Do not print status for stereo_tri point cloud center calculation. Will 
        # print the status later for triangulation for each tile.
        resetStatus = True
        updateNumDoneTiles(out_prefix, prog, resetStatus)
        
    if opt.verbose:
        print('%s' % ' '.join(call))
    try:
        status = subprocess.call(call)
    except OSError as e:
        raise Exception('%s: %s' % (binpath, e))
    if status != 0:
        raise Exception('Stereo step ' + kw['msg'] + ' failed')

    if status == 0 and prog != "stereo_tri":
        resetStatus = False
        updateNumDoneTiles(out_prefix, prog, resetStatus)

def keepOnlySpecified(keep_only, out_prefix, subdirs, settings):
    """
    Merge all VRTs and wipe any files the user does not want to keep.
    """
    
    if keep_only == "unchanged":
      return # keep everything unchanged
    elif keep_only == "essential":
      keep_only = "L.txt R.txt .exr -L.tif -F.tif disp-diff.tif -PC.tif"
    
    # List all the files stereo may create. We will not attempt to delete
    # any files except these (and subdirs). In particular log files in the 
    # top-most directory are not deleted.
    all_files = set()
    for ext in ".vwip .match cropped.tif -L.tif -R.tif ask.tif L.txt R.txt .exr sub.tif -D.tif -RD.tif -B.tif disp-diff.tif -F.tif -PC.tif".split():
        for f in glob.glob(out_prefix + "*" + ext):
            all_files.add(f)

    # List all the files with the suffixes the user wants to keep. Must also keep
    # the aux.xml files, as those can have CRS information.    
    keep_files = set()
    if keep_only == "all_combined":
            keep_files = all_files.copy()
    else:
        for keep in keep_only.split():
            for f in glob.glob(out_prefix + "*" + keep) + \
                     glob.glob(out_prefix + "*" + keep + ".aux.xml"): 
                keep_files.add(f)

    if len(keep_files) == 0:
        # Sometimes the user can make a mistake here. If not sure, don't do anything.
        print("No files to keep. This is likely a user error. Not deleting any files.")
        return
        
    for f in keep_files:
    
        if int(settings['correlator_mode'][0]) != 0 and 'PC.tif' in f:
           # No PC.tif in correlator-mode 
           continue

        print("Keeping: " + f)
        if isVrt(f):
            f_merged = os.path.splitext(f)[0] + "_merged.tif"
            print("Convert " + f + " from VRT to TIF format.")
            cmd = ['gdal_translate', '-co', 'compress=lzw', '-co', 'TILED=yes', '-co',
                   'INTERLEAVE=BAND', '-co', 'BLOCKXSIZE=256', '-co', 'BLOCKYSIZE=256',
                   '-co', 'BIGTIFF=IF_SAFER', f, f_merged]
            (out, err, status) = asp_system_utils.executeCommand(cmd, realTimeOutput = True)
            if status != 0:
                raise Exception('Failed converting VRT to TIF. Qutting. ' + \
                                'Likely this run is partially wiped now.')
                
            print("Renaming: " + f_merged + " to " + f)
            shutil.move(f_merged, f)
            
            # If _merged.aux.xml exists, rename it to .xml. This is for 3D CRS.
            merged_aux = f_merged + ".aux.xml"
            aux = f + ".aux.xml"
            if os.path.exists(merged_aux):
                print("Renaming: " + merged_aux + " to " + aux)
                shutil.move(merged_aux, aux)

    # Delete the other files and directories
    for f in list(all_files) + subdirs:
        if f not in keep_files:
            print("Deleting: " + f)
            if os.path.isdir(f):
                shutil.rmtree(f)
            else:
                os.remove(f)
    
    # It looks useful to print the final cloud, as the messages above
    # can be quite verbose.
    pc_file = glob.glob(out_prefix + "-PC.tif")
    if len(pc_file) > 0 and int(settings['correlator_mode'][0]) == 0:
        print("Output point cloud: " + pc_file[0])

def set_collar_size(new_collar_size, args, settings):
    """
    Set collar_size to a new value, and update the settings and args.
    """
    
    collar_size = new_collar_size
    asp_cmd_utils.set_option(args, '--sgm-collar-size', [str(collar_size)])
    settings['collar_size'] = [str(collar_size)]
    return (settings, args)

def parallel_stereo_args(p, opt, parallel_args, args):
    """
    Find the options used by parallel_stereo which are not
    passed to the stereo executables. This is fragile.
    """

    # Map from option, say --job-size-w, to the variable name, say, job_size_w.
    opt2var = {}
    for x in p._actions:
        # For cases like ['-l', '--long'] get the second entry
        opt_str = getattr(x, 'option_strings')[-1]
        opt2var[opt_str] = getattr(x, 'dest')

    extra_args = []
    option_dict = vars(opt) # Each option understood by python, and its parsed value
    for arg in parallel_args[1:]:

        if arg in args:
            # This is an option passed to stereo
            continue

        if len(arg) >= 2 and arg[0] == '-' and arg[1] != '-':
            # We expect two dashes
            raise Exception('Failed to parse: ' + arg)

        if not arg in opt2var.keys():
            # Does not start with dash
            continue 

        var = opt2var[arg]

        if option_dict[var] is None:
            # This option was not specified
            continue

        if isinstance(option_dict[var], bool):
            # This is a bool flag, append it with no value
            extra_args.append(arg)
            continue

        if (isinstance(option_dict[var], int)    or
            isinstance(option_dict[var], float)  or
            asp_string_utils.isString(option_dict[var])):
            # For int, float, and string, append together with value
            extra_args.append(arg)
            extra_args.append(str(option_dict[var]))
            continue

        # We cannot currently parse other options than int, string, float, and bool.
        # Should be simple to fix if need be. 
        raise Exception('Could not parse: ' + arg)

    return extra_args

def setup_run_multiview(opt, settings, parallel_args, args):
    '''
    Prepare and run multiview stereo. Triangulation is joint, but
    the steps before that are done separately for each pair.
    '''
 
    # TODO(oalexan1): This function calls parallel_stereo again. This is
    # confusing.
    
    if opt.keep_only == "all_combined":
        # Change the default to "unchanged"
        opt.keep_only = "unchanged"
    if opt.keep_only != "unchanged":
        # If the user specified something else
        raise Exception('Option --keep-only does not work with multiview stereo.')

    # TODO(oalexan1): avoid confusing the logic below
    asp_cmd_utils.wipe_option(parallel_args, '-s', 1)
    if os.path.exists(opt.stereo_file):
        parallel_args.extend(['--stereo-file', opt.stereo_file])

    # Args only for parallel_stereo
    extra_args = parallel_stereo_args(p, opt, parallel_args, args)

    if opt.entry_point < Step.tri:
        run_multiview(__file__, args, extra_args, opt.entry_point,
                        opt.stop_point, opt.verbose, settings)
        # Everything is done.
        sys.exit(0)
    else:
        # We will arrive here after this script invokes itself
        # for multiview.  Set up the directories. After this
        # exits, triangulation will run.
        create_symlinks_for_multiview(opt, args, settings)
                
if __name__ == '__main__':
    usage = '''parallel_stereo [options] <images> [<cameras>]
                  <output_file_prefix> [DEM]
        Extensions are automatically added to the output files.
        Camera model arguments may be optional for some stereo
        session types (e.g. isis). Stereo parameters should be
        set in the stereo.default file.\n''' + asp_system_utils.get_asp_version()

    # What makes this program different from stereo is that it
    # tries to treat ASP as a multi-process system instead of a
    # multi-threaded executable. This has benefits on the super
    # computer by allowing a single stereo pair use multiple
    # computers. It also allows us to get past the single-threaded
    # constraints of ISIS.

    # Algorithm: When this script is started, it calls itself on each node if
    # doing steps 1, 2, or 4 (corr, rfne, tri). Those scripts in turn start
    # actual jobs on those nodes. For the other steps, the script does the work
    # itself.

    # Python does not know how to disambiguate between two options which start
    # with a similar prefix.
    if '--threads' in sys.argv:
        print("Ignoring the option --threads. Use --threads-multiprocess " + \
              "and --threads-singleprocess.")
        asp_cmd_utils.wipe_option(sys.argv, '--threads', 1)

    p = argparse.ArgumentParser(usage=usage)
    p.add_argument('--nodes-list', dest='nodes_list', default=None,
        help='The list of computing nodes, one per line. If not provided, run ' + \
        'on the local machine.')
    p.add_argument('--ssh', dest='ssh', default=None,
        help='The path to the ssh program to use to connect to other nodes')
    p.add_argument('--processes', dest='processes', default=None, type=int,
        help='The number of processes to use per node.')
    p.add_argument('--threads-multiprocess', dest='threads_multi', default=None, type=int,
        help='The number of threads to use per process when running multiple ' + \
        'processes.')
    p.add_argument('--threads-singleprocess', dest='threads_single', default=None, type=int,
        help='The number of threads to use when running a single process (PPRC ' + \
        'and FLTR).')
    p.add_argument('--corr-seed-mode', dest='seed_mode', default=None, type=int,
        help='Correlation seed strategy. See stereo_corr for options.')
    p.add_argument('-e', '--entry-point', dest='entry_point', default=0, type=int,
        help='Stereo Pipeline entry point (an integer from 0-6).')
    p.add_argument('--stop-point', dest='stop_point', default=7, type=int,
        help='Stereo Pipeline stop point (an integer from 1-7). Stop before this ' + \
        'step.')
    p.add_argument('--job-size-w', dest='job_size_w', default=-1, type=int,
        help='Pixel width of input image tile for a single process. Set ' + \
        'automatically.')
    p.add_argument('--job-size-h', dest='job_size_h', default=-1, type=int,
        help='Pixel height of input image tile for a single process. Set ' + \
        'automatically.')
    p.add_argument('--sgm-collar-size', dest='sgm_collar_size', default=-1, type=int,
        help='The padding around each tile to process. Set automatically.')
    p.add_argument('--resume-at-corr', dest='resume_at_corr', default=False,
        action='store_true',
        help='Start at the correlation stage and skip recomputing the valid low ' + \
        'and full-res disparities for that stage. Do not change ' + \
        '--left-image-crop-win, etc., when running this.')
    p.add_argument('--prev-run-prefix', dest='prev_run_prefix', default=None,
        help='Start at the triangulation stage while reusing the data from this ' + \
        'prefix. The new run can use different cameras, bundle adjustment prefix, ' + \
        'or bathy planes (if applicable). Do not change crop windows, as that ' + \
        'would invalidate the run.')
    p.add_argument('--keep-only', dest='keep_only', default="all_combined",
        help='If set to "all_combined", which is the default, at the end of a ' + \
        'successful run combine the results from subdirectories into .tif files ' + \
        'with the given output prefix, and delete the subdirectories. If set to ' + \
        '"essential", keep only PC.tif and the files needed to recreate it (those ' + \
        'ending with L.txt, R.txt, .exr, L.tif, -F.tif). If set to "unchanged", ' + \
        'keep the run directory as it is. For fine-grained control, specify a ' + \
        'quoted list of suffixes of files to keep, such as "L.txt R.txt .exr ' + \
        '.match -L.tif -PC.tif".')
    p.add_argument('--sparse-disp-options', dest='sparse_disp_options',
        help='Options to pass directly to sparse_disp. Use quotes around this ' + \
        'string.')
    p.add_argument('-v', '--version', dest='version', default=False, action='store_true',
        help='Display the version of software.')
    p.add_argument('-s', '--stereo-file', dest='stereo_file', default='./stereo.default',
        help='Explicitly specify the stereo.default file to use. ' + \
        '[default: ./stereo.default]')
    p.add_argument('--verbose', dest='verbose', default=False, action='store_true',
        help='Display the commands being executed.')
    p.add_argument('--parallel-options', dest='parallel_options', default='--sshdelay 0.2',
        help='Options to pass directly to GNU Parallel.')
    # Internal variables below.
    # The id of the tile to process, 0 <= tile_id < num_tiles.
    p.add_argument('--tile-id', dest='tile_id', default=None, type=int,
                   help=argparse.SUPPRESS)
    # Directory where the job is running
    p.add_argument('--work-dir', dest='work_dir', default=None,
                   help=argparse.SUPPRESS)
    # ISIS settings
    p.add_argument('--isisroot', dest='isisroot', default=None,
                   help=argparse.SUPPRESS)
    p.add_argument('--isisdata', dest='isisdata', default=None,
                   help=argparse.SUPPRESS)
    # Debug option
    p.add_argument('--dry-run', dest='dryrun', default=False, action='store_true',
                   help="Do not launch the jobs, only print the commands that should be run.")

    (opt, args) = p.parse_known_args()
    args = asp_cmd_utils.clean_args(args)
    
    if opt.version:
        asp_system_utils.print_version_and_exit()

    if not args and not opt.version:
        p.print_help()
        die('\nERROR: Missing input files', code=2)

    # Ensure our 'parallel' is not out of date
    asp_system_utils.check_parallel_version()

    if opt.tile_id is None and opt.resume_at_corr:
        print("Resuming at the correlation stage.")
        opt.entry_point = Step.corr
        if opt.stop_point <= Step.corr:
            raise Exception('Cannot resume at correlation if --stop-point ' + \
                            'is set to stop before that.')

    if opt.threads_single is None:
        opt.threads_single = get_num_cpus()

    # If corr-seed-mode was not specified, read it from the file
    if opt.seed_mode is None:
        opt.seed_mode = parse_corr_seed_mode(opt.stereo_file)
    # If not set in the file either, use 1.
    if opt.seed_mode is None:
        opt.seed_mode = 1
    # Pass it to the subprocesses
    args.extend(['--corr-seed-mode', str(opt.seed_mode)])

    if os.path.exists(opt.stereo_file):
        args.extend(['--stereo-file', opt.stereo_file])

    if opt.tile_id is None:
        # When the script is started, set some options from the
        # environment which we will pass to the scripts we spawn
        # 1. Set the work directory
        opt.work_dir = os.getcwd()
        # 2. Set the ISIS settings if any
        if 'ISISROOT' in os.environ: opt.isisroot = os.environ['ISISROOT']
        if 'ISISDATA' in os.environ: opt.isisdata = os.environ['ISISDATA']
    else:
        # After the script spawns itself to nodes, it starts in the
        # home dir. Make it go to the right place.
        os.chdir(opt.work_dir)
        # Set the ISIS settings
        if opt.isisroot  is not None: os.environ['ISISROOT' ] = opt.isisroot
        if opt.isisdata is not None: os.environ['ISISDATA'] = opt.isisdata

    # This command needs to be run after we switch to the work directory,
    # hence no earlier than this point.
    sep = ","
    settings = run_and_parse_output("stereo_parse", args, sep, opt.verbose)
    out_prefix = settings['out_prefix'][0]

    if settings['stereo_algorithm'][0].lower() == 'libelas' and \
       platform.system() == "Darwin" and platform.machine() == "arm64":
       raise Exception('The stereo algorithm ' + \
                        settings['stereo_algorithm'][0] + ' is not supported on Mac Arm.')

    # In the master process, need to create the list of nodes. Must happen
    # after we are in the work dir and have out_prefix. This ensures
    # the list is not in a temp dir of one of the nodes.
    if opt.tile_id is None and opt.nodes_list is not None:
        if not os.path.isfile(opt.nodes_list):
            die('\nERROR: No such nodes-list file: ' + opt.nodes_list, code=2)
        local_nodes_list = out_prefix + "-nodes-list.txt"
        mkdir_p(os.path.dirname(local_nodes_list))
        if opt.nodes_list != local_nodes_list:
            shutil.copy2(opt.nodes_list, local_nodes_list)
            opt.nodes_list = local_nodes_list
        asp_cmd_utils.wipe_option(sys.argv, '--nodes-list', 1)
        sys.argv.extend(['--nodes-list', opt.nodes_list])
        print("Nodes list: " + opt.nodes_list)
      
    # Padded tiles are needed if later we do blending 
    using_padded_tiles = use_padded_tiles(settings)

    # By default use 8 threads for SGM/MGM. Then use proportionately fewer
    # processes to not run out of RAM. Otherwise see the function
    # get_best_procs_threads().
    # TODO(oalexan1): This logic needs to be all in one place.
    # For ISIS needs to use more processes in triangulation.
    alg = stereo_alg_to_num(settings['stereo_algorithm'][0])
    if alg > VW_CORRELATION_BM and alg < VW_CORRELATION_OTHER:
        if opt.threads_multi is None:
            opt.threads_multi = 8
        if opt.processes is None:
            num_cpus = get_num_cpus()
            opt.processes = int(float(num_cpus) / float(opt.threads_multi) + 0.5)
            if opt.processes <= 0:
                opt.processes = 1

    if opt.version:
        args.append('-v')

    # TODO(oalexan1): This block must be a function called setup_job_size().
    if int(opt.job_size_h < 0) + int(opt.job_size_w < 0) == 1:
        raise Exception('Must set neither or both of --job-size-h and --job-size-w.')
    # If --job-size-h and --job-size-w are not set, set them depending on the algorithm
    localEpi = (settings['alignment_method'][0] == 'local_epipolar')
    if opt.job_size_h < 0:
        if alg >= VW_CORRELATION_OTHER or localEpi:
            # Local alignment needs small tiles
            opt.job_size_w = 512
            opt.job_size_h = 512
        elif alg == VW_CORRELATION_BM:
            # Regular block matching
            opt.job_size_w = 2048
            opt.job_size_h = 2048
        elif alg > VW_CORRELATION_BM:
            # SGM, MGM, etc.
            opt.job_size_w = 2048
            opt.job_size_h = 2048
    if opt.sgm_collar_size < 0:
        if alg >= VW_CORRELATION_OTHER or localEpi:
            # Local alignment
            opt.sgm_collar_size = 128
        elif alg == VW_CORRELATION_BM:
            # Regular block matching
            opt.sgm_collar_size = 0
        elif alg > VW_CORRELATION_BM:
            # SGM, MGM, etc.
            opt.sgm_collar_size = 256
    # Update the collar size in the settings and args
    (settings, args) = set_collar_size(opt.sgm_collar_size, args, settings)

    # A string unlikely to be in the output. This should be in sync with stereo_parse.
    sep2 = '--non-comma-separator--'
    
    georef=run_and_parse_output("stereo_parse", args, sep2, opt.verbose)
    georef["WKT"] = "".join(georef["WKT"])
    georef["GeoTransform"] = "".join(georef["GeoTransform"])

    print ('Using tiles (before collar addition) of ' + str(opt.job_size_w) + \
           ' x ' + str(opt.job_size_h) + ' pixels.')
    print('Using a collar (padding) for each tile of ' + settings['collar_size'][0] + \
          ' pixels.')
    
    # parallel_args is what we will pass to spawned copies of parallel_stereo,
    # while 'args' is what we pass to plain stereo.
    parallel_args = sys.argv # shallow copy
    
    # Individual tools may use these options, but then they confuse parallel_stereo.
    # They are usually encountered in log files and can be passed in by mistake.
    if '--compute-point-cloud-center-only' in parallel_args:
      raise Exception('Option --compute-point-cloud-center-only is not ' + \
                      'supported in parallel_stereo. Use stereo_tri instead.')
    if '--compute-low-res-disparity-only' in parallel_args:
        raise Exception('Option --compute-low-res-disparity-only is not ' + \
                        'supported in parallel_stereo. Use stereo_corr instead.')

    # See if to resume at triangulation. This logic must happen after we figured
    # if we need padded tiles, otherwise the bookkeeping will be wrong.
    if opt.tile_id is None and opt.prev_run_prefix is not None:
        print("Starting at the triangulation stage while reusing a previous run.")
        opt.entry_point = Step.tri
        if opt.stop_point <= Step.tri:
            raise Exception('Cannot resume at triangulation if --stop-point ' + \
                            'is set to stop before that.')
        sym_link_prev_run(opt.prev_run_prefix, out_prefix)

        # Now that we have data, update the settings value
        settings = run_and_parse_output("stereo_parse", args, sep, opt.verbose)

        # Create the directory tree for this run
        subdirs = create_subdirs_symlink(opt, args, settings)

    # TODO(oalexan1): The giant block below needs to be broken up into two
    # functions, called main_run() and and tile_run(). Careful testing will be
    # needed, including for multiview stereo.
    if opt.tile_id is None:

        # We get here when the script is started. The current running process
        # has become the management process that spawns other copies of itself
        # on other machines. This block will only do actual work when we hit a
        # non-multiprocess step like pprc or fltr.

        # Wipe options which we will override.
        asp_cmd_utils.wipe_option(parallel_args, '-e', 1)
        asp_cmd_utils.wipe_option(parallel_args, '--entry-point', 1)
        asp_cmd_utils.wipe_option(parallel_args, '--stop-point', 1)
        
        num_pairs = int(settings['num_stereo_pairs'][0])
        if num_pairs > 1:
            # Multiview stereo needs special treatment
            setup_run_multiview(opt, settings, parallel_args, args)

        # Preprocessing
        step = Step.pprc
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()
            normal_run('stereo_pprc', opt, args, out_prefix, msg='%d: Preprocessing' % step)
            # Now that L.tif is saved, recompute trans_left_image_size.
            settings = run_and_parse_output("stereo_parse", args, sep, opt.verbose)

        # Correlation
        step = Step.corr
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()

            # Do low-res correlation, this happens just once.
            calc_lowres_disp(args, opt, sep, resume = opt.resume_at_corr)

            # symlink D_sub, D_sub_spread, etc.
            subdirs = create_subdirs_symlink(opt, args, settings)

            # Run full-res stereo using multiple processes.
            check_system_memory(opt, args, settings)
            parallel_args.extend(['--skip-low-res-disparity-comp'])
            spawn_to_nodes(step, opt, settings, parallel_args, subdirs)
            # Low-res disparity is done, so wipe that option
            asp_cmd_utils.wipe_option(parallel_args, '--skip-low-res-disparity-comp', 0)
            
            # Bugfix: When doing refinement for a given tile, we must see
            # the result of correlation for all tiles. To achieve that,
            # rename all correlation tiles to something else,
            # build the vrt of all correlation tiles, and sym link
            # that vrt from all tile directories.
            rename_files(settings, subdirs, "-D.tif", "-Dnosym.tif")
            build_vrt('stereo_corr', opt, args, settings, georef, 
                      "-D.tif", "-Dnosym.tif", 
                      contract_tiles = using_padded_tiles)
            subdirs = create_subdirs_symlink(opt, args, settings) # symlink D.tif

        skip_refine_step = (int(settings['subpixel_mode'][0]) > 6)

        # Blending (when using local_epipolar alignment, or SGM/MGM, or external algorithms)
        step = Step.blend
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()
            
            if not using_padded_tiles:
                print("No work to do at the blending step.")
            else:
                subdirs = create_subdirs_symlink(opt, args, settings)
                spawn_to_nodes(step, opt, settings, parallel_args, subdirs)

                if not skip_refine_step:
                    # Do the same trick as after stereo_corr
                    rename_files(settings, subdirs, "-B.tif", "-Bnosym.tif")
                    build_vrt('stereo_blend', opt, args, settings, georef, 
                              "-B.tif", "-Bnosym.tif", 
                              contract_tiles = False)
                    create_subdirs_symlink(opt, args, settings) # symlink B.tif

            # At the blending step, make a vrt from the per-tile lr-disp differences.
            if settings['save_lr_disp_diff'][0] != '0':
                if using_padded_tiles:
                    build_vrt('stereo_blend', opt, args, settings, georef,
                            "-L-R-disp-diff.tif", "-L-R-disp-diff-blend.tif")
                else:
                    build_vrt('stereo_blend', opt, args, settings, georef,
                            "-L-R-disp-diff.tif", "-L-R-disp-diff.tif")

        # Refinement
        step = Step.rfne
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()
            # If we used SGM/MGM or some other method making use of
            # tiles, need to read from the blend file.
            if using_padded_tiles:
                parallel_args.extend(['--subpix-from-blend'])
            if not skip_refine_step:
                subdirs = create_subdirs_symlink(opt, args, settings)
                spawn_to_nodes(step, opt, settings, parallel_args, subdirs)

            try:
                build_vrt('stereo_rfne', opt, args, settings, georef, "-RD.tif", "-RD.tif")
            except Exception as e:
                # Make the error message more informative
                raise Exception('Failed to build a VRT from */*RD.tif files. Must redo at least the refinement step. Additional error message: ' + str(e))
                
        # Filtering
        step = Step.fltr
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()
                
            normal_run('stereo_fltr', opt, args, out_prefix, msg='%d: Filtering' % step)
            create_subdirs_symlink(opt, args, settings) # symlink F.tif

        # Triangulation
        # TODO(oalexan1): For stereo we do not need to pad the tiles, which should
        # be a nice speed up. But then must ensure both the padded and non-padded
        # tiles are wiped after the run is done.
        step = Step.tri
        subdirs = []
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()

            # TODO(oalexan1): Must wipe the subdirs in all cases.
            if int(settings['correlator_mode'][0]) != 0 and \
                '--num-matches-from-disparity' not in parallel_args and \
                '--num-matches-from-disp-triplets' not in parallel_args:
                # Skip triangulation, but continue further down with cleanup.
                print("Skipping triangulation with --correlator-mode.")
            else:
                # Compute the cloud center. Done once per run.
                tmp_args = args[:] # deep copy
                tmp_args.append('--compute-point-cloud-center-only')
                normal_run('stereo_tri', opt, tmp_args, out_prefix, 
                           msg='%d: Triangulation' % step)
                # Point cloud center computation was done
                parallel_args.extend(['--skip-point-cloud-center-comp'])

                # Wipe options that should have been done when the
                # the center of point cloud was computed
                asp_cmd_utils.wipe_option(parallel_args, '--unalign-disparity', 0)
                asp_cmd_utils.wipe_option(parallel_args, '--num-matches-from-disparity', 1)
                asp_cmd_utils.wipe_option(parallel_args, '--num-matches-from-disp-triplets', 1)
                
                # Create subdirs and symlink the files just created
                subdirs = create_subdirs_symlink(opt, args, settings)
                # Run triangulation on multiple machines
                spawn_to_nodes(step, opt, settings, parallel_args, subdirs)
                if int(settings['correlator_mode'][0]) == 0:
                  build_vrt('stereo_tri', opt, args, settings, georef, "-PC.tif", "-PC.tif")

        # If the run concluded successfully, merge and wipe
        step = Step.clean
        subdirs = []
        if (opt.entry_point <= step):
            if (opt.stop_point <= step):
                sys.exit()
        if (opt.stop_point > Step.clean):
          if len(subdirs) == 0:
            # If this was not created by now, do it now
            subdirs = create_subdirs_symlink(opt, args, settings)
          keepOnlySpecified(opt.keep_only, out_prefix, subdirs, settings)
                
       # End main process case
    else:

        # This process was spawned by GNU Parallel with a given
        # value of opt.tile_id. Launch the job for that tile.
        if opt.verbose:
            print("Running on machine: ", os.uname())

        try:
            # Pick the tile we want from the list of tiles
            tiles = readTiles(out_prefix)
            tile  = tiles[opt.tile_id]
            prog_name = stereoProgName(opt.entry_point)

            if (opt.entry_point == Step.corr):
                check_system_memory(opt, args, settings)
                tile_run(prog_name, args, opt, settings, tile,
                         msg='%d: Correlation' % opt.entry_point)

            if (opt.entry_point == Step.blend):
                tile_run(prog_name, args, opt, settings, tile,
                         msg='%d: Blending' % opt.entry_point)

            if (opt.entry_point == Step.rfne):
                tile_run(prog_name, args, opt, settings, tile,
                         msg='%d: Refinement' % opt.entry_point)

            if (opt.entry_point == Step.tri):
                tile_run(prog_name, args, opt, settings, tile,
                         msg='%d: Triangulation' % opt.entry_point)

        except Exception as e:
            die(e)
            raise
