    @Override
    public int run(String[] args) throws Exception {

        Logger.getLogger(TypeRegistry.class).setLevel(Level.ALL);

        ca.setThreshold(Level.INFO);
        log.addAppender(ca);
        log.setLevel(Level.INFO);

        // Initialize the markings file helper so we get the right markings file
        MarkingFunctions.Factory.createMarkingFunctions();
        TypeRegistry.reset();

        // Parse the job arguments
        Configuration conf = parseArguments(args, this.getConf());

        if (conf == null) {
            printUsage();
            return -1;
        }

        updateConfWithOverrides(conf);
        AccumuloHelper cbHelper = new AccumuloHelper();
        cbHelper.setup(conf);

        TypeRegistry.getInstance(conf);

        log.error(conf.toString());
        log.error(String.format("getStrings('%s') = %s", TypeRegistry.INGEST_DATA_TYPES, conf.get(TypeRegistry.INGEST_DATA_TYPES)));
        log.error(String.format("getStrings('data.name') = %s", conf.get("data.name")));
        int index = 0;
        for (String name : TypeRegistry.getTypeNames()) {
            log.error(String.format("name[%d] = '%s'", index++, name));
        }

        if (TypeRegistry.getTypes().isEmpty()) {
            log.error("No data types were configured");
            return -1;
        }

        if (!registerTableNames(conf)) {
            return -1;
        }

        boolean wasConfigureTablesSuccessful = configureTables(cbHelper, conf);
        if (!wasConfigureTablesSuccessful) {
            return -1;
        } else if (createTablesOnly) {
            // Exit early if we are only creating tables
            log.info("Created tables: " + getTables(conf) + " successfully!");
            return 0;
        }

        try {
            serializeAggregatorConfiguration(cbHelper, conf, log);
        } catch (TableNotFoundException tnf) {
            log.error("One or more configured DataWave tables are missing in Accumulo. If this is a new system or if new tables have recently been introduced, run a job using the '-createTablesOnly' flag before attempting to ingest more data",
                            tnf);
            return -1;
        }

        // get the source and output hadoop file systems
        FileSystem inputFs = getFileSystem(conf, srcHdfs);
        FileSystem outputFs = (writeDirectlyToDest ? getFileSystem(conf, destHdfs) : inputFs);
        conf.set("output.fs.uri", outputFs.getUri().toString());

        // get the qualified work directory path
        Path unqualifiedWorkPath = Path.getPathWithoutSchemeAndAuthority(new Path(workDir));
        conf.set("ingest.work.dir.unqualified", unqualifiedWorkPath.toString());
        Path workDirPath = getFileSystem(conf, (writeDirectlyToDest ? destHdfs : srcHdfs)).makeQualified(unqualifiedWorkPath);
        conf.set("ingest.work.dir.qualified", workDirPath.toString());

        // Create the Job
        Job job = Job.getInstance(conf);
        // Job copies the configuration, so any changes made after this point don't get captured in the job.
        // Use the job's configuration from this point.
        conf = job.getConfiguration();
        if (!useMapOnly || !outputMutations) {
            // Calculate the sampled splits, splits file, and set up the partitioner, but not if only doing only a map phase and outputting mutations
            // if not outputting mutations and only doing a map phase, we still need to go through this logic as the MultiRFileOutputFormatter
            // depends on this.
            try {
                configureBulkPartitionerAndOutputFormatter(job, cbHelper, conf, outputFs);
            } catch (Exception e) {
                log.error(e);
                return -1;
            }
        }

        job.setJarByClass(this.getClass());
        for (Path inputPath : getFilesToProcess(inputFs, inputFileLists, inputFileListMarker, inputPaths)) {
            FileInputFormat.addInputPath(job, inputPath);
        }
        for (Path dependency : jobDependencies)
            job.addFileToClassPath(dependency);

        configureInputFormat(job, cbHelper, conf);

        configureJob(job, conf, workDirPath, outputFs);

        // Log configuration
        log.info("Types: " + TypeRegistry.getTypeNames());
        log.info("Tables: " + Arrays.toString(tableNames));
        log.info("InputFormat: " + job.getInputFormatClass().getName());
        log.info("Mapper: " + job.getMapperClass().getName());
        log.info("Reduce tasks: " + (useMapOnly ? 0 : reduceTasks));
        log.info("Split File: " + workDirPath + "/splits.txt");

        // Note that if we run any other jobs in the same vm (such as a sampler), then we may
        // need to catch and throw away an exception here
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(conf));

        startDaemonProcesses(conf);
        long start = System.currentTimeMillis();
        job.submit();
        JobID jobID = job.getJobID();
        log.info("JOB ID: " + jobID);

        createFileWithRetries(outputFs, new Path(workDirPath, jobID.toString()));

        // Wait for reduce progress to pass the 30% mark and then
        // kick off the next job of this type.
        boolean done = false;
        while (generateMarkerFile && !done && !job.isComplete()) {
            if (job.reduceProgress() > markerFileReducePercentage) {
                File flagDir = new File(flagFileDir);
                if (flagDir.isDirectory()) {
                    // Find flag files that start with this datatype
                    RegexFileFilter filter;
                    if (flagFilePattern != null) {
                        filter = new RegexFileFilter(flagFilePattern);
                    } else {
                        filter = new RegexFileFilter(outputMutations ? ".*_(live|fivemin)_.*\\.flag" : ".*_(bulk|onehr)_.*\\.flag");
                    }
                    File[] flagFiles = flagDir.listFiles((FilenameFilter) filter);
                    if (flagFiles.length > 0) {
                        // Reverse sort by time to get the earliest file
                        Comparator<File> comparator = LastModifiedFileComparator.LASTMODIFIED_COMPARATOR;
                        if (!markerFileFIFO) {
                            comparator = LastModifiedFileComparator.LASTMODIFIED_REVERSE;
                        }
                        Arrays.sort(flagFiles, comparator);
                        // Just grab the first one and rename it to .marker
                        File flag = flagFiles[0];
                        File targetFile = new File(flag.getAbsolutePath() + (pipelineId == null ? "" : '.' + pipelineId) + ".marker");
                        if (!flag.renameTo(targetFile)) {
                            log.error("Unable to rename flag file: " + flag.getAbsolutePath());
                            continue;
                        }
                        log.info("Renamed flag file " + flag + " to " + targetFile);
                    } else {
                        log.info("No more flag files to process");
                        // + datatype);
                    }
                } else {
                    log.error("Flag file directory does not exist: " + flagFileDir);
                }
                done = true;
            } else {
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException ie) {
                    // do nothing
                }
            }
        }

        job.waitForCompletion(true);
        long stop = System.currentTimeMillis();

        // output the counters to the log
        Counters counters = job.getCounters();
        log.info(counters);

        JobClient jobClient = new JobClient((org.apache.hadoop.mapred.JobConf) job.getConfiguration());
        RunningJob runningJob = jobClient.getJob(new org.apache.hadoop.mapred.JobID(jobID.getJtIdentifier(), jobID.getId()));

        // If the job failed, then don't bring the map files online.
        if (!job.isSuccessful()) {
            return jobFailed(job, runningJob, outputFs, workDirPath);
        }

        // determine if we had processing errors
        if (counters.findCounter(IngestProcess.RUNTIME_EXCEPTION).getValue() > 0) {
            eventProcessingError = true;
            log.error("Found Runtime Exceptions in the counters");
        }
        if (counters.findCounter(IngestInput.EVENT_FATAL_ERROR).getValue() > 0) {
            eventProcessingError = true;
            log.error("Found Fatal Errors in the counters");
        }

        // If we're doing "live" ingest (sending mutations to accumulo rather than
        // bringing map files online), then simply delete the workDir since it
        // doesn't contain anything we need. If we are doing bulk ingest, then
        // write out a marker file to indicate that the job is complete and a
        // separate process will bulk import the map files.
        if (outputMutations) {
            markFilesLoaded(inputFs, FileInputFormat.getInputPaths(job));
            boolean deleted = outputFs.delete(workDirPath, true);
            if (!deleted) {
                log.error("Unable to remove job working directory: " + workDirPath);
            }
        } else {
            // now move the job directory over to the warehouse if needed
            FileSystem destFs = getFileSystem(conf, destHdfs);

            if (!inputFs.equals(destFs) && !writeDirectlyToDest) {
                Configuration distCpConf = conf;
                // Use the configuration dir specified on the command-line for DistCP if necessary.
                // Basically this means pulling in all of the *-site.xml config files from the specified
                // directory. By adding these resources last, their properties will override those in the
                // current config.
                if (distCpConfDir != null) {
                    distCpConf = new Configuration(false);
                    FilenameFilter ff = (dir, name) -> name.toLowerCase().endsWith("-site.xml");
                    for (String file : new File(distCpConfDir).list(ff)) {
                        Path path = new Path(distCpConfDir, file);
                        distCpConf.addResource(file.replace("-site", "-default"));
                        distCpConf.addResource(path);
                    }
                }

                log.info("Moving (using distcp) " + unqualifiedWorkPath + " from " + inputFs.getUri() + " to " + destFs.getUri());
                try {
                    distCpDirectory(unqualifiedWorkPath, inputFs, destFs, distCpConf, deleteAfterDistCp);
                } catch (Exception e) {
                    log.error("Failed to move job directory over to the warehouse.", e);
                    return -3;
                }
            }

            Path destWorkDirPath = FileSystem.get(destHdfs, conf).makeQualified(unqualifiedWorkPath);
            boolean marked = markJobComplete(destFs, destWorkDirPath);
            if (!marked) {
                log.error("Failed to create marker file indicating job completion.");
                return -3;
            }
        }

        // if we had a failure writing the metrics, or we have event processing errors, then return -5
        // this should result in administrators getting an email, but the job will be considered successful

        if (metricsOutputEnabled) {
            log.info("Writing Stats");
            Path statsDir = new Path(unqualifiedWorkPath.getParent(), "IngestMetrics");
            if (!writeStats(log, job, jobID, counters, start, stop, outputMutations, inputFs, statsDir, this.metricsLabelOverride)) {
                log.warn("Failed to output statistics for the job");
                return -5;
            }
        } else {
            log.info("Ingest stats output disabled via 'ingestMetricsDisabled' flag");
        }

        if (eventProcessingError) {
            log.warn("Job had processing errors.  See counters for more information");
            return -5;
        }

        return 0;
    }

    @Override
    public int run(String[] args) throws Exception {

        Logger.getLogger(TypeRegistry.class).setLevel(Level.ALL);

        ca.setThreshold(Level.INFO);
        log.addAppender(ca);
        log.setLevel(Level.INFO);

        // Initialize the markings file helper so we get the right markings file
        MarkingFunctions.Factory.createMarkingFunctions();
        TypeRegistry.reset();

        // Parse the job arguments
        Configuration conf = parseArguments(args, this.getConf());

        if (conf == null) {
            printUsage();
            return -1;
        }

        updateConfWithOverrides(conf);
        AccumuloHelper cbHelper = new AccumuloHelper();
        cbHelper.setup(conf);

        TypeRegistry.getInstance(conf);

        log.error(conf.toString());
        log.error(String.format("getStrings('%s') = %s", TypeRegistry.INGEST_DATA_TYPES, conf.get(TypeRegistry.INGEST_DATA_TYPES)));
        log.error(String.format("getStrings('data.name') = %s", conf.get("data.name")));
        int index = 0;
        for (String name : TypeRegistry.getTypeNames()) {
            log.error(String.format("name[%d] = '%s'", index++, name));
        }

        if (TypeRegistry.getTypes().isEmpty()) {
            log.error("No data types were configured");
            return -1;
        }

        if (!registerTableNames(conf)) {
            return -1;
        }

        boolean wasConfigureTablesSuccessful = configureTables(cbHelper, conf);
        if (!wasConfigureTablesSuccessful) {
            return -1;
        } else if (createTablesOnly) {
            // Exit early if we are only creating tables
            log.info("Created tables: " + getTables(conf) + " successfully!");
            return 0;
        }

        try {
            serializeAggregatorConfiguration(cbHelper, conf, log);
        } catch (TableNotFoundException tnf) {
            log.error("One or more configured DataWave tables are missing in Accumulo. If this is a new system or if new tables have recently been introduced, run a job using the '-createTablesOnly' flag before attempting to ingest more data",
                            tnf);
            return -1;
        }

        // get the source and output hadoop file systems
        FileSystem inputFs = getFileSystem(conf, srcHdfs);
        FileSystem outputFs = (writeDirectlyToDest ? getFileSystem(conf, destHdfs) : inputFs);
        conf.set("output.fs.uri", outputFs.getUri().toString());

        // get the qualified work directory path
        Path unqualifiedWorkPath = Path.getPathWithoutSchemeAndAuthority(new Path(workDir));
        conf.set("ingest.work.dir.unqualified", unqualifiedWorkPath.toString());
        Path workDirPath = getFileSystem(conf, (writeDirectlyToDest ? destHdfs : srcHdfs)).makeQualified(unqualifiedWorkPath);
        conf.set("ingest.work.dir.qualified", workDirPath.toString());

        // Create the Job
        Job job = Job.getInstance(conf);
        // Job copies the configuration, so any changes made after this point don't get captured in the job.
        // Use the job's configuration from this point.
        conf = job.getConfiguration();
        if (!useMapOnly || !outputMutations) {
            // Calculate the sampled splits, splits file, and set up the partitioner, but not if only doing only a map phase and outputting mutations
            // if not outputting mutations and only doing a map phase, we still need to go through this logic as the MultiRFileOutputFormatter
            // depends on this.
            try {
                configureBulkPartitionerAndOutputFormatter(job, cbHelper, conf, outputFs);
            } catch (Exception e) {
                log.error(e);
                return -1;
            }
        }

        job.setJarByClass(this.getClass());
        for (Path inputPath : getFilesToProcess(inputFs, inputFileLists, inputFileListMarker, inputPaths)) {
            FileInputFormat.addInputPath(job, inputPath);
        }
        for (Path dependency : jobDependencies)
            job.addFileToClassPath(dependency);

        configureInputFormat(job, cbHelper, conf);

        configureJob(job, conf, workDirPath, outputFs);

        // Log configuration
        log.info("Types: " + TypeRegistry.getTypeNames());
        log.info("Tables: " + Arrays.toString(tableNames));
        log.info("InputFormat: " + job.getInputFormatClass().getName());
        log.info("Mapper: " + job.getMapperClass().getName());
        log.info("Reduce tasks: " + (useMapOnly ? 0 : reduceTasks));
        log.info("Split File: " + workDirPath + "/splits.txt");

        // Note that if we run any other jobs in the same vm (such as a sampler), then we may
        // need to catch and throw away an exception here
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(conf));

        startDaemonProcesses(conf);
        long start = System.currentTimeMillis();
        job.submit();
        JobID jobID = job.getJobID();
        log.info("JOB ID: " + jobID);

        createFileWithRetries(outputFs, new Path(workDirPath, jobID.toString()));

        // Wait for reduce progress to pass the 30% mark and then
        // kick off the next job of this type.
        boolean done = false;
        while (generateMarkerFile && !done && !job.isComplete()) {
            if (job.reduceProgress() > markerFileReducePercentage) {
                File flagDir = new File(flagFileDir);
                if (flagDir.isDirectory()) {
                    // Find flag files that start with this datatype
                    RegexFileFilter filter;
                    if (flagFilePattern != null) {
                        filter = new RegexFileFilter(flagFilePattern);
                    } else {
                        filter = new RegexFileFilter(outputMutations ? ".*_(live|fivemin)_.*\\.flag" : ".*_(bulk|onehr)_.*\\.flag");
                    }
                    File[] flagFiles = flagDir.listFiles((FilenameFilter) filter);
                    if (flagFiles.length > 0) {
                        // Reverse sort by time to get the earliest file
                        Comparator<File> comparator = LastModifiedFileComparator.LASTMODIFIED_COMPARATOR;
                        if (!markerFileFIFO) {
                            comparator = LastModifiedFileComparator.LASTMODIFIED_REVERSE;
                        }
                        Arrays.sort(flagFiles, comparator);
                        // Just grab the first one and rename it to .marker
                        File flag = flagFiles[0];
                        File targetFile = new File(flag.getAbsolutePath() + (pipelineId == null ? "" : '.' + pipelineId) + ".marker");
                        if (!flag.renameTo(targetFile)) {
                            log.error("Unable to rename flag file: " + flag.getAbsolutePath());
                            continue;
                        }
                        log.info("Renamed flag file " + flag + " to " + targetFile);
                    } else {
                        log.info("No more flag files to process");
                        // + datatype);
                    }
                } else {
                    log.error("Flag file directory does not exist: " + flagFileDir);
                }
                done = true;
            } else {
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException ie) {
                    // do nothing
                }
            }
        }

        job.waitForCompletion(true);
        long stop = System.currentTimeMillis();

        // output the counters to the log
        Counters counters = job.getCounters();
        log.info(counters);

        JobClient jobClient = new JobClient((org.apache.hadoop.mapred.JobConf) job.getConfiguration());
        RunningJob runningJob = jobClient.getJob(new org.apache.hadoop.mapred.JobID(jobID.getJtIdentifier(), jobID.getId()));

        // If the job failed, then don't bring the map files online.
        if (!job.isSuccessful()) {
            return jobFailed(job, runningJob, outputFs, workDirPath);
        }

        // determine if we had processing errors
        if (counters.findCounter(IngestProcess.RUNTIME_EXCEPTION).getValue() > 0) {
            eventProcessingError = true;
            log.error("Found Runtime Exceptions in the counters");
        }
        if (counters.findCounter(IngestInput.EVENT_FATAL_ERROR).getValue() > 0) {
            eventProcessingError = true;
            log.error("Found Fatal Errors in the counters");
        }

        // If we're doing "live" ingest (sending mutations to accumulo rather than
        // bringing map files online), then simply delete the workDir since it
        // doesn't contain anything we need. If we are doing bulk ingest, then
        // write out a marker file to indicate that the job is complete and a
        // separate process will bulk import the map files.
        if (outputMutations) {
            markFilesLoaded(inputFs, FileInputFormat.getInputPaths(job));
            boolean deleted = outputFs.delete(workDirPath, true);
            if (!deleted) {
                log.error("Unable to remove job working directory: " + workDirPath);
            }
        } else {
            // now move the job directory over to the warehouse if needed
            FileSystem destFs = getFileSystem(conf, destHdfs);

            if (!inputFs.equals(destFs) && !writeDirectlyToDest) {
                Configuration distCpConf = conf;
                // Use the configuration dir specified on the command-line for DistCP if necessary.
                // Basically this means pulling in all of the *-site.xml config files from the specified
                // directory. By adding these resources last, their properties will override those in the
                // current config.
                if (distCpConfDir != null) {
                    distCpConf = new Configuration(false);
                    FilenameFilter ff = new FilenameFilter() {
                        @Override
                        public boolean accept(File dir, String name) {
                            return name.toLowerCase().endsWith("-site.xml");
                        }
                    };
                    for (String file : new File(distCpConfDir).list(ff)) {
                        Path path = new Path(distCpConfDir, file);
                        distCpConf.addResource(file.replace("-site", "-default"));
                        distCpConf.addResource(path);
                    }
                }

                log.info("Moving (using distcp) " + unqualifiedWorkPath + " from " + inputFs.getUri() + " to " + destFs.getUri());
                try {
                    distCpDirectory(unqualifiedWorkPath, inputFs, destFs, distCpConf, deleteAfterDistCp);
                } catch (Exception e) {
                    log.error("Failed to move job directory over to the warehouse.", e);
                    return -3;
                }
            }

            Path destWorkDirPath = FileSystem.get(destHdfs, conf).makeQualified(unqualifiedWorkPath);
            boolean marked = markJobComplete(destFs, destWorkDirPath);
            if (!marked) {
                log.error("Failed to create marker file indicating job completion.");
                return -3;
            }
        }

        // if we had a failure writing the metrics, or we have event processing errors, then return -5
        // this should result in administrators getting an email, but the job will be considered successful

        if (metricsOutputEnabled) {
            log.info("Writing Stats");
            Path statsDir = new Path(unqualifiedWorkPath.getParent(), "IngestMetrics");
            if (!writeStats(log, job, jobID, counters, start, stop, outputMutations, inputFs, statsDir, this.metricsLabelOverride)) {
                log.warn("Failed to output statistics for the job");
                return -5;
            }
        } else {
            log.info("Ingest stats output disabled via 'ingestMetricsDisabled' flag");
        }

        if (eventProcessingError) {
            log.warn("Job had processing errors.  See counters for more information");
            return -5;
        }

        return 0;
    }

Replace Anonymous With Lambda datawave.ingest.mapreduce.job.IngestJob.run.ff.new FilenameFilter with (dir, name) -> name.toLowerCase().endsWith("-site.xml") in method public run(args String[]) : int from class datawave.ingest.mapreduce.job.IngestJob


    private TreeMap<Text,String> reverseSortByShardIds(TreeMap<Text,String> shardIdToLocations) {
        // drop the dates after today's date
        TreeMap<Text,String> shardIdsToTservers = Maps.newTreeMap((o1, o2) -> o2.compareTo(o1));
        shardIdsToTservers.putAll(shardIdToLocations);
        return shardIdsToTservers;
    }

    private TreeMap<Text,String> reverseSortByShardIds(TreeMap<Text,String> shardIdToLocations) {
        // drop the dates after today's date
        TreeMap<Text,String> shardIdsToTservers = Maps.newTreeMap(new Comparator<Text>() {
            @Override
            public int compare(Text o1, Text o2) {
                return o2.compareTo(o1);
            }
        });
        shardIdsToTservers.putAll(shardIdToLocations);
        return shardIdsToTservers;
    }

Replace Anonymous With Lambda datawave.ingest.mapreduce.partition.BalancedShardPartitioner.reverseSortByShardIds.shardIdsToTservers.newTreeMap.new Comparator<Text> with (o1, o2) -> o2.compareTo(o1) in method private reverseSortByShardIds(shardIdToLocations TreeMap<Text,String>) : TreeMap<Text,String> from class datawave.ingest.mapreduce.partition.BalancedShardPartitioner


    /**
     * Write the flag file. This is done in several steps to ensure we can easily recover if we are killed somewhere in-between.
     *
     * 1) move the files to the flagging directory for those that do not already exist in flagging, flagged, or loaded 2) create the flag.generating file for
     * those files we were able to move, and do not exist elsewhere 3) set the timestamp of the flag file to that of the most recent file 4) move the flagging
     * files to the flagged directory 5) move the generating file to the final flag file form 6) create the cleanup file
     *
     * Using these steps, a cleanup of an abnormal termination is as follows: 1) move all flagging files to the base directory 2) for all flag.generating files,
     * move the flagged files to the base directory 3) remove the flag.generating files.
     *
     * @param fc
     * @param inFiles
     * @throws IOException
     */
    void writeFlagFile(final FlagDataTypeConfig fc, Collection<InputFile> inFiles) throws IOException {

        long estSize = getFlagFileSize(fc, inFiles);
        if (inFiles == null || inFiles.isEmpty())
            throw new IllegalArgumentException("inFiles for Flag file");
        final ConcurrentHashMap<InputFile,Path> moved = new ConcurrentHashMap<>();
        File flagFile = null;
        final FileSystem fs = getHadoopFS();
        try {
            // first lets create the dest directories, and move the files into the flagging directory
            final AtomicLong latestTime = new AtomicLong(-1);
            List<Callable<Path>> callables = Lists.newArrayList();
            for (final InputFile inFile : inFiles) {
                // Create the flagging directory
                Callable<Path> mover = () -> {
                    Path dstFlagging = getDestPath(inFile, "flagging", fc);
                    if (directoryCache.getIfPresent(dstFlagging) == null && !fs.exists(dstFlagging.getParent())) {
                        fs.mkdirs(dstFlagging.getParent());
                        directoryCache.put(dstFlagging, dstFlagging);
                    }
                    // Create the flagged directory
                    Path dstFlagged = getDestPath(inFile, "flagged", fc);
                    if (directoryCache.getIfPresent(dstFlagged) == null && !fs.exists(dstFlagged.getParent())) {
                        fs.mkdirs(dstFlagged.getParent());
                        directoryCache.put(dstFlagged, dstFlagged);
                    }
                    // Check for existence of the file already in the flagging, flagged, or loaded directories
                    Path dstLoaded = getDestPath(inFile, "loaded", fc);
                    if (fs.exists(dstFlagged) || fs.exists(dstFlagging) || fs.exists(dstLoaded)) {
                        log.warn("Unable to move file " + inFile.getDirectory() + "/" + inFile.getFileName()
                                        + " as it already exists in the flagging, flagged, or loaded directory");
                    } else {
                        // now move the file into the flagging directory
                        if (fs.rename(inFile.getPath(), dstFlagging)) {
                            moved.put(inFile, dstFlagging);
                            latestTime.set(Math.max(inFile.getTimestamp(), latestTime.get()));
                        } else {
                            log.error("Unable to move file " + inFile.getDirectory() + "/" + inFile.getFileName() + ", skipping");
                        }
                    }
                    return dstLoaded;
                };
                // can look at I files that have a logical ts greater than the logical ts of the last compaction. that would indicate what has been imported
                // since that compaction started. which is basically the pile up we've seen.
                // we can look at the size of that compacted file and use that to determine how to size the next one

                // two prior compactions: 3139106321 3139418030, (311k) apart. and 4,274,086,297 4.2GB
                // 3139017948 - 3138938475 = 79473 (22 minutes) 3822549588 bytes = 3.8GB 3138796228 to 3139468770
                // can determine how many I files came in since the most recent C file
                // can find 5 most recent logical changes

                callables.add(mover);
            }
            try {
                List<Future<Path>> execResults = executor.invokeAll(callables);

                for (Future<Path> future : execResults) {
                    if (future.get() == null)
                        throw new IOException("Error while attempting to create path");
                }

            } catch (InterruptedException e) {
                throw new IOException(e.getCause());
            } catch (ExecutionException e) {
                throw new IOException(e.getCause());
            }

            // if no files moved, then abort
            if (moved.isEmpty()) {
                log.warn("No pending files were able to be moved to the flagging directory. Please investigate.");
                return;
            }

            // create the flag.generating file
            Path first = moved.entrySet().iterator().next().getValue();
            long now = System.currentTimeMillis();
            String baseName = fmc.getFlagFileDirectory() + File.separator + df.format(now / 1000) + "_" + fc.getIngestPool() + "_" + fc.getDataName() + "_"
                            + first.getName() + "+" + moved.size();
            log.info("Creating flag file" + baseName + ".flag" + " for data type " + fc.getDataName() + " containing " + moved.size() + " files");
            File f = new File(baseName + ".flag.generating");
            if (f.createNewFile()) {
                flagFile = f;
            } else {
                throw new IOException("Unable to create flag file " + f);
            }
            FileOutputStream flagOS = new FileOutputStream(f);
            StringBuilder sb = new StringBuilder(fmc.getDatawaveHome() + File.separator + fc.getScript());
            if (fc.getFileListMarker() == null) {
                String sep = " ";
                for (InputFile inFile : moved.keySet()) {
                    if (fc.isCollectMetrics())
                        ctx.getCounter(InputFile.class.getSimpleName(), inFile.getFileName()).setValue(inFile.getTimestamp());
                    Path dstFlagged = getDestPath(inFile, "flagged", fc);
                    sb.append(sep).append(dstFlagged.toUri());
                    sep = ",";
                }
            } else {
                // put a variable in here instead which will resolve to the flag file inprogress when that time comes. The baseName could have changed by then.
                sb.append(" ${JOB_FILE}");
            }
            sb.append(" ").append(fc.getReducers()).append(" -inputFormat ").append(fc.getInputFormat().getName()).append(" ");
            if (fc.getFileListMarker() != null) {
                sb.append("-inputFileLists -inputFileListMarker ").append(fc.getFileListMarker()).append(" ");
            }
            if (fc.getExtraIngestArgs() != null) {
                sb.append(fc.getExtraIngestArgs());
            }
            sb.append("\n");
            if (fc.getFileListMarker() != null) {
                sb.append(fc.getFileListMarker()).append('\n');
                for (InputFile inFile : moved.keySet()) {
                    if (fc.isCollectMetrics())
                        ctx.getCounter(InputFile.class.getSimpleName(), inFile.getFileName()).setValue(inFile.getTimestamp());
                    Path dstFlagged = getDestPath(inFile, "flagged", fc);
                    sb.append(dstFlagged.toUri()).append('\n');
                }
            }
            if (estSize != sb.length()) {
                log.error("Estimated size is not accurate: " + estSize + " vs " + sb.length());
            }

            flagOS.write(sb.toString().getBytes());
            flagOS.close();

            // now set the modification time of the flag file
            if (fmc.isSetFlagFileTimestamp()) {
                f.setLastModified(latestTime.get());
            }

            // Now we have files moved into the flagging directory, and a flag.generating file created

            // move the files to the flagged directory
            callables = Lists.newArrayList();

            for (Map.Entry<InputFile,Path> entry : moved.entrySet()) {
                final InputFile inFile = entry.getKey();
                final Path dstFlagging = entry.getValue();
                final Path dstFlagged = getDestPath(inFile, "flagged", fc);
                Callable<Path> mover = () -> {
                    if (fs.rename(dstFlagging, dstFlagged)) {
                        moved.put(inFile, dstFlagged);
                        if (fc.isCollectMetrics())
                            ctx.getCounter(FlagFile.class.getSimpleName(), dstFlagged.getName()).setValue(System.currentTimeMillis());
                    } else {
                        throw new IOException("Unable to move file " + inFile.getDirectory() + "/" + inFile.getFileName());
                    }

                    return dstFlagged;
                };
                callables.add(mover);
            }

            try {
                List<Future<Path>> execResults = executor.invokeAll(callables);

                for (Future<Path> future : execResults) {
                    if (future.get() == null)
                        throw new IOException("Error while attempting to create path");
                }

            } catch (InterruptedException e) {
                throw new IOException(e.getCause());
            } catch (ExecutionException e) {
                throw new IOException(e.getCause());
            }

            File f2 = new File(baseName + ".flag");
            if (f.renameTo(f2)) {
                flagFile = f2;
            } else {
                throw new IOException("Failed to rename" + f.toString() + " to " + f2);
            }

            try {
                lock.lock();
                // after we write a file, set the timeout to the forceInterval
                fc.setLast(now + fc.getTimeoutMilliSecs());
            } finally {
                lock.unlock();
            }

            if (fc.isCollectMetrics())
                updateStats();

            if (fc.isCollectMetrics()) {
                writeMetrics(ctx.getReporter(), this.fmc.getFlagMetricsDirectory(), new Path(baseName).getName(), ct, cc);
            }
        } catch (IOException ex) {
            log.error("Unable to complete flag file ", ex);
            moveFilesBack(moved);
            if (flagFile != null) {
                flagFile.delete();
            }
            throw ex;
        }
    }

    /**
     * Write the flag file. This is done in several steps to ensure we can easily recover if we are killed somewhere in-between.
     *
     * 1) move the files to the flagging directory for those that do not already exist in flagging, flagged, or loaded 2) create the flag.generating file for
     * those files we were able to move, and do not exist elsewhere 3) set the timestamp of the flag file to that of the most recent file 4) move the flagging
     * files to the flagged directory 5) move the generating file to the final flag file form 6) create the cleanup file
     *
     * Using these steps, a cleanup of an abnormal termination is as follows: 1) move all flagging files to the base directory 2) for all flag.generating files,
     * move the flagged files to the base directory 3) remove the flag.generating files.
     *
     * @param fc
     * @param inFiles
     * @throws IOException
     */
    void writeFlagFile(final FlagDataTypeConfig fc, Collection<InputFile> inFiles) throws IOException {

        long estSize = getFlagFileSize(fc, inFiles);
        if (inFiles == null || inFiles.isEmpty())
            throw new IllegalArgumentException("inFiles for Flag file");
        final ConcurrentHashMap<InputFile,Path> moved = new ConcurrentHashMap<>();
        File flagFile = null;
        final FileSystem fs = getHadoopFS();
        try {
            // first lets create the dest directories, and move the files into the flagging directory
            final AtomicLong latestTime = new AtomicLong(-1);
            List<Callable<Path>> callables = Lists.newArrayList();
            for (final InputFile inFile : inFiles) {
                // Create the flagging directory
                Callable<Path> mover = new Callable<Path>() {
                    @Override
                    public Path call() throws IOException {
                        Path dstFlagging = getDestPath(inFile, "flagging", fc);
                        if (directoryCache.getIfPresent(dstFlagging) == null && !fs.exists(dstFlagging.getParent())) {
                            fs.mkdirs(dstFlagging.getParent());
                            directoryCache.put(dstFlagging, dstFlagging);
                        }
                        // Create the flagged directory
                        Path dstFlagged = getDestPath(inFile, "flagged", fc);
                        if (directoryCache.getIfPresent(dstFlagged) == null && !fs.exists(dstFlagged.getParent())) {
                            fs.mkdirs(dstFlagged.getParent());
                            directoryCache.put(dstFlagged, dstFlagged);
                        }
                        // Check for existence of the file already in the flagging, flagged, or loaded directories
                        Path dstLoaded = getDestPath(inFile, "loaded", fc);
                        if (fs.exists(dstFlagged) || fs.exists(dstFlagging) || fs.exists(dstLoaded)) {
                            log.warn("Unable to move file " + inFile.getDirectory() + "/" + inFile.getFileName()
                                            + " as it already exists in the flagging, flagged, or loaded directory");
                        } else {
                            // now move the file into the flagging directory
                            if (fs.rename(inFile.getPath(), dstFlagging)) {
                                moved.put(inFile, dstFlagging);
                                latestTime.set(Math.max(inFile.getTimestamp(), latestTime.get()));
                            } else {
                                log.error("Unable to move file " + inFile.getDirectory() + "/" + inFile.getFileName() + ", skipping");
                            }
                        }
                        return dstLoaded;
                    }
                };
                // can look at I files that have a logical ts greater than the logical ts of the last compaction. that would indicate what has been imported
                // since that compaction started. which is basically the pile up we've seen.
                // we can look at the size of that compacted file and use that to determine how to size the next one

                // two prior compactions: 3139106321 3139418030, (311k) apart. and 4,274,086,297 4.2GB
                // 3139017948 - 3138938475 = 79473 (22 minutes) 3822549588 bytes = 3.8GB 3138796228 to 3139468770
                // can determine how many I files came in since the most recent C file
                // can find 5 most recent logical changes

                callables.add(mover);
            }
            try {
                List<Future<Path>> execResults = executor.invokeAll(callables);

                for (Future<Path> future : execResults) {
                    if (future.get() == null)
                        throw new IOException("Error while attempting to create path");
                }

            } catch (InterruptedException e) {
                throw new IOException(e.getCause());
            } catch (ExecutionException e) {
                throw new IOException(e.getCause());
            }

            // if no files moved, then abort
            if (moved.isEmpty()) {
                log.warn("No pending files were able to be moved to the flagging directory. Please investigate.");
                return;
            }

            // create the flag.generating file
            Path first = moved.entrySet().iterator().next().getValue();
            long now = System.currentTimeMillis();
            String baseName = fmc.getFlagFileDirectory() + File.separator + df.format(now / 1000) + "_" + fc.getIngestPool() + "_" + fc.getDataName() + "_"
                            + first.getName() + "+" + moved.size();
            log.info("Creating flag file" + baseName + ".flag" + " for data type " + fc.getDataName() + " containing " + moved.size() + " files");
            File f = new File(baseName + ".flag.generating");
            if (f.createNewFile()) {
                flagFile = f;
            } else {
                throw new IOException("Unable to create flag file " + f);
            }
            FileOutputStream flagOS = new FileOutputStream(f);
            StringBuilder sb = new StringBuilder(fmc.getDatawaveHome() + File.separator + fc.getScript());
            if (fc.getFileListMarker() == null) {
                String sep = " ";
                for (InputFile inFile : moved.keySet()) {
                    if (fc.isCollectMetrics())
                        ctx.getCounter(InputFile.class.getSimpleName(), inFile.getFileName()).setValue(inFile.getTimestamp());
                    Path dstFlagged = getDestPath(inFile, "flagged", fc);
                    sb.append(sep).append(dstFlagged.toUri());
                    sep = ",";
                }
            } else {
                // put a variable in here instead which will resolve to the flag file inprogress when that time comes. The baseName could have changed by then.
                sb.append(" ${JOB_FILE}");
            }
            sb.append(" ").append(fc.getReducers()).append(" -inputFormat ").append(fc.getInputFormat().getName()).append(" ");
            if (fc.getFileListMarker() != null) {
                sb.append("-inputFileLists -inputFileListMarker ").append(fc.getFileListMarker()).append(" ");
            }
            if (fc.getExtraIngestArgs() != null) {
                sb.append(fc.getExtraIngestArgs());
            }
            sb.append("\n");
            if (fc.getFileListMarker() != null) {
                sb.append(fc.getFileListMarker()).append('\n');
                for (InputFile inFile : moved.keySet()) {
                    if (fc.isCollectMetrics())
                        ctx.getCounter(InputFile.class.getSimpleName(), inFile.getFileName()).setValue(inFile.getTimestamp());
                    Path dstFlagged = getDestPath(inFile, "flagged", fc);
                    sb.append(dstFlagged.toUri()).append('\n');
                }
            }
            if (estSize != sb.length()) {
                log.error("Estimated size is not accurate: " + estSize + " vs " + sb.length());
            }

            flagOS.write(sb.toString().getBytes());
            flagOS.close();

            // now set the modification time of the flag file
            if (fmc.isSetFlagFileTimestamp()) {
                f.setLastModified(latestTime.get());
            }

            // Now we have files moved into the flagging directory, and a flag.generating file created

            // move the files to the flagged directory
            callables = Lists.newArrayList();

            for (Map.Entry<InputFile,Path> entry : moved.entrySet()) {
                final InputFile inFile = entry.getKey();
                final Path dstFlagging = entry.getValue();
                final Path dstFlagged = getDestPath(inFile, "flagged", fc);
                Callable<Path> mover = new Callable<Path>() {
                    @Override
                    public Path call() throws Exception {
                        if (fs.rename(dstFlagging, dstFlagged)) {
                            moved.put(inFile, dstFlagged);
                            if (fc.isCollectMetrics())
                                ctx.getCounter(FlagFile.class.getSimpleName(), dstFlagged.getName()).setValue(System.currentTimeMillis());
                        } else {
                            throw new IOException("Unable to move file " + inFile.getDirectory() + "/" + inFile.getFileName());
                        }

                        return dstFlagged;
                    }
                };
                callables.add(mover);
            }

            try {
                List<Future<Path>> execResults = executor.invokeAll(callables);

                for (Future<Path> future : execResults) {
                    if (future.get() == null)
                        throw new IOException("Error while attempting to create path");
                }

            } catch (InterruptedException e) {
                throw new IOException(e.getCause());
            } catch (ExecutionException e) {
                throw new IOException(e.getCause());
            }

            File f2 = new File(baseName + ".flag");
            if (f.renameTo(f2)) {
                flagFile = f2;
            } else {
                throw new IOException("Failed to rename" + f.toString() + " to " + f2);
            }

            try {
                lock.lock();
                // after we write a file, set the timeout to the forceInterval
                fc.setLast(now + fc.getTimeoutMilliSecs());
            } finally {
                lock.unlock();
            }

            if (fc.isCollectMetrics())
                updateStats();

            if (fc.isCollectMetrics()) {
                writeMetrics(ctx.getReporter(), this.fmc.getFlagMetricsDirectory(), new Path(baseName).getName(), ct, cc);
            }
        } catch (IOException ex) {
            log.error("Unable to complete flag file ", ex);
            moveFilesBack(moved);
            if (flagFile != null) {
                flagFile.delete();
            }
            throw ex;
        }
    }

Replace Anonymous With Lambda datawave.util.flag.FlagMaker.writeFlagFile.mover.new Callable<Path> with () -> {Path dstFlagging=getDestPath(inFile,"flagging",fc); in method package writeFlagFile(fc FlagDataTypeConfig, inFiles Collection<InputFile>) : void from class datawave.util.flag.FlagMaker


    public Comparator<S> getComparator(boolean lifo) {
        if (lifo) {
            return (o1, o2) -> {
                // return the reverse order
                return Ordering.natural().compare(o2, o1);
            };
        } else {
            return Ordering.natural();
        }
    }

    public Comparator<S> getComparator(boolean lifo) {
        if (lifo) {
            return new Comparator<S>() {
                @Override
                public int compare(S o1, S o2) {
                    // return the reverse order
                    return Ordering.natural().compare(o2, o1);
                }
            };
        } else {
            return Ordering.natural();
        }
    }

Replace Anonymous With Lambda datawave.util.flag.processor.AbstractSliceDistributor.getComparator.new Comparator<S> with (o1, o2) -> {return Ordering.natural().compare(o2,o1); in method public getComparator(lifo boolean) : Comparator<S> from class datawave.util.flag.processor.AbstractSliceDistributor


    @Override
    public MimeDecoder createMimeDecoder() {
        return b -> MimeUtility.decodeText(new String(b, "iso-8859-1")).getBytes();
    }

    @Override
    public MimeDecoder createMimeDecoder() {
        return new MimeDecoder() {
            @Override
            public byte[] decode(byte[] b) throws IOException {
                return MimeUtility.decodeText(new String(b, "iso-8859-1")).getBytes();
            }
        };
    }

Replace Anonymous With Lambda datawave.ingest.config.TestIngestConfigurationImpl.createMimeDecoder.new MimeDecoder with b -> MimeUtility.decodeText(new String(b,"iso-8859-1")).getBytes() in method public createMimeDecoder() : MimeDecoder from class datawave.ingest.config.TestIngestConfigurationImpl


    private HttpServer createFileServer(String path, int port) throws Exception {
        final String resp = readFile(path);
        HttpServer server = HttpServer.create(new InetSocketAddress(port), 0);

        server.setExecutor(null);
        server.createContext("/", e -> {
            e.sendResponseHeaders(200, 0);
            e.getResponseHeaders().set("Content-Type", "text/xml");

            OutputStream responseBody = e.getResponseBody();
            responseBody.write(resp.getBytes());
            responseBody.close();
        });

        return server;
    }

    private HttpServer createFileServer(String path, int port) throws Exception {
        final String resp = readFile(path);
        HttpServer server = HttpServer.create(new InetSocketAddress(port), 0);

        server.setExecutor(null);
        server.createContext("/", new HttpHandler() {
            @Override
            public void handle(HttpExchange e) throws IOException {
                e.sendResponseHeaders(200, 0);
                e.getResponseHeaders().set("Content-Type", "text/xml");

                OutputStream responseBody = e.getResponseBody();
                responseBody.write(resp.getBytes());
                responseBody.close();
            }
        });

        return server;
    }

Replace Anonymous With Lambda datawave.ingest.data.config.FieldConfigHelperTest.createFileServer.createContext.new HttpHandler with e -> {e.sendResponseHeaders(200,0); in method private createFileServer(path String, port int) : HttpServer from class datawave.ingest.data.config.FieldConfigHelperTest


    protected Job createMockJob() {

        Job mocked = PowerMock.createMock(Job.class);

        mocked.getConfiguration();
        EasyMock.expectLastCall().andAnswer(this::createMockConfiguration).anyTimes();

        PowerMock.replay(mocked);

        return mocked;
    }

    protected Job createMockJob() {

        Job mocked = PowerMock.createMock(Job.class);

        mocked.getConfiguration();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                return createMockConfiguration();
            }

        }).anyTimes();

        PowerMock.replay(mocked);

        return mocked;
    }

Replace Anonymous With Lambda datawave.ingest.mapreduce.job.CBMutationOutputFormatterTest.createMockJob.anyTimes.andAnswer.new IAnswer<Object> with this::createMockConfiguration in method protected createMockJob() : Job from class datawave.ingest.mapreduce.job.CBMutationOutputFormatterTest


    protected Configuration createMockConfiguration() {

        MultiRFileOutputFormatterTest.mockedConfiguration.clear();

        Configuration mocked = PowerMock.createMock(Configuration.class);

        MultiRFileOutputFormatterTest.logger.info(String.format("createMockConfiguration: %d", mocked.hashCode()));

        mocked.set(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            String key = (String) EasyMock.getCurrentArguments()[0];
            String value = (String) EasyMock.getCurrentArguments()[1];

            MultiRFileOutputFormatterTest.mockedConfiguration.put(key, value);

            return null;
        }).anyTimes();

        mocked.setStrings(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            if (2 <= EasyMock.getCurrentArguments().length) {

                String key = (String) EasyMock.getCurrentArguments()[0];
                String[] values = new String[EasyMock.getCurrentArguments().length - 1];

                for (int index = 1; index <= values.length; index++) {

                    values[index - 1] = (String) EasyMock.getCurrentArguments()[index];
                }

                MultiRFileOutputFormatterTest.mockedConfiguration.put(key, StringUtils.arrayToString(values));
            }
            return null;
        }).anyTimes();

        mocked.get(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            String key = (String) EasyMock.getCurrentArguments()[0];
            String value = (String) EasyMock.getCurrentArguments()[1];

            if (MultiRFileOutputFormatterTest.mockedConfiguration.containsKey(key)) {

                value = MultiRFileOutputFormatterTest.mockedConfiguration.get(key);
            }

            return value;
        }).anyTimes();

        PowerMock.replay(mocked);

        return mocked;
    }

    protected Configuration createMockConfiguration() {

        MultiRFileOutputFormatterTest.mockedConfiguration.clear();

        Configuration mocked = PowerMock.createMock(Configuration.class);

        MultiRFileOutputFormatterTest.logger.info(String.format("createMockConfiguration: %d", mocked.hashCode()));

        mocked.set(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                String key = (String) EasyMock.getCurrentArguments()[0];
                String value = (String) EasyMock.getCurrentArguments()[1];

                MultiRFileOutputFormatterTest.mockedConfiguration.put(key, value);

                return null;
            }

        }).anyTimes();

        mocked.setStrings(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                if (2 <= EasyMock.getCurrentArguments().length) {

                    String key = (String) EasyMock.getCurrentArguments()[0];
                    String[] values = new String[EasyMock.getCurrentArguments().length - 1];

                    for (int index = 1; index <= values.length; index++) {

                        values[index - 1] = (String) EasyMock.getCurrentArguments()[index];
                    }

                    MultiRFileOutputFormatterTest.mockedConfiguration.put(key, StringUtils.arrayToString(values));
                }
                return null;
            }

        }).anyTimes();

        mocked.get(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                String key = (String) EasyMock.getCurrentArguments()[0];
                String value = (String) EasyMock.getCurrentArguments()[1];

                if (MultiRFileOutputFormatterTest.mockedConfiguration.containsKey(key)) {

                    value = MultiRFileOutputFormatterTest.mockedConfiguration.get(key);
                }

                return value;
            }
        }).anyTimes();

        PowerMock.replay(mocked);

        return mocked;
    }

Replace Anonymous With Lambda datawave.ingest.mapreduce.job.MultiRFileOutputFormatterTest.createMockConfiguration.anyTimes.andAnswer.new IAnswer<Object> with () -> {if(2 <= EasyMock.getCurrentArguments().length){String key=(String)EasyMock.getCurrentArguments()[0]; in method protected createMockConfiguration() : Configuration from class datawave.ingest.mapreduce.job.MultiRFileOutputFormatterTest


    private void testConcurrentCommitTaskWithSubDir(int version) throws Exception {
        final Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        final Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);

        conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
        FileSystem.closeAll();

        final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        final FileOutputCommitter amCommitter = new SafeFileOutputCommitter(outDir, jContext);
        amCommitter.setupJob(jContext);

        final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
        taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
        taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);

        final TextOutputFormat[] tof = new TextOutputFormat[2];
        for (int i = 0; i < tof.length; i++) {
            tof[i] = new TextOutputFormat() {
                @Override
                public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
                    final FileOutputCommitter foc = (FileOutputCommitter) getOutputCommitter(context);
                    return new Path(new Path(foc.getWorkPath(), SUB_DIR), getUniqueFile(context, getOutputName(context), extension));
                }
            };
        }

        final ExecutorService executor = Executors.newFixedThreadPool(2);
        try {
            for (int i = 0; i < taCtx.length; i++) {
                final int taskIdx = i;
                executor.submit((Callable<Void>) () -> {
                    final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
                    outputCommitter.setupTask(taCtx[taskIdx]);
                    final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
                    writeOutput(rw, taCtx[taskIdx]);
                    outputCommitter.commitTask(taCtx[taskIdx]);
                    return null;
                });
            }
        } finally {
            executor.shutdown();
            while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
                LOG.info("Awaiting thread termination!");
            }
        }

        amCommitter.commitJob(jContext);
        final RawLocalFileSystem lfs = new RawLocalFileSystem();
        lfs.setConf(conf);
        assertFalse("Must not end up with sub_dir/sub_dir", lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));

        // validate output
        validateContent(OUT_SUB_DIR);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

    private void testConcurrentCommitTaskWithSubDir(int version) throws Exception {
        final Job job = Job.getInstance();
        FileOutputFormat.setOutputPath(job, outDir);
        final Configuration conf = job.getConfiguration();
        conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
        conf.setInt(FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version);

        conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
        FileSystem.closeAll();

        final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
        final FileOutputCommitter amCommitter = new SafeFileOutputCommitter(outDir, jContext);
        amCommitter.setupJob(jContext);

        final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
        taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
        taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);

        final TextOutputFormat[] tof = new TextOutputFormat[2];
        for (int i = 0; i < tof.length; i++) {
            tof[i] = new TextOutputFormat() {
                @Override
                public Path getDefaultWorkFile(TaskAttemptContext context, String extension) throws IOException {
                    final FileOutputCommitter foc = (FileOutputCommitter) getOutputCommitter(context);
                    return new Path(new Path(foc.getWorkPath(), SUB_DIR), getUniqueFile(context, getOutputName(context), extension));
                }
            };
        }

        final ExecutorService executor = Executors.newFixedThreadPool(2);
        try {
            for (int i = 0; i < taCtx.length; i++) {
                final int taskIdx = i;
                executor.submit(new Callable<Void>() {
                    @Override
                    public Void call() throws IOException, InterruptedException {
                        final OutputCommitter outputCommitter = tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
                        outputCommitter.setupTask(taCtx[taskIdx]);
                        final RecordWriter rw = tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
                        writeOutput(rw, taCtx[taskIdx]);
                        outputCommitter.commitTask(taCtx[taskIdx]);
                        return null;
                    }
                });
            }
        } finally {
            executor.shutdown();
            while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
                LOG.info("Awaiting thread termination!");
            }
        }

        amCommitter.commitJob(jContext);
        final RawLocalFileSystem lfs = new RawLocalFileSystem();
        lfs.setConf(conf);
        assertFalse("Must not end up with sub_dir/sub_dir", lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));

        // validate output
        validateContent(OUT_SUB_DIR);
        FileUtil.fullyDelete(new File(outDir.toString()));
    }

Replace Anonymous With Lambda datawave.ingest.mapreduce.job.SafeFileOutputCommitterTest.testConcurrentCommitTaskWithSubDir.submit.new Callable<Void> with () -> {final OutputCommitter outputCommitter=tof[taskIdx].getOutputCommitter(taCtx[taskIdx]); in method private testConcurrentCommitTaskWithSubDir(version int) : void from class datawave.ingest.mapreduce.job.SafeFileOutputCommitterTest


        protected Logger createMockLogger() {

            Logger log = PowerMock.createMock(Logger.class);

            if (null == debugMessages) {

                debugMessages = new ArrayList<>();
            }

            if (null == infoMessages) {

                infoMessages = new ArrayList<>();
            }

            log.debug(EasyMock.anyObject(String.class));
            EasyMock.expectLastCall().andAnswer(() -> {

                String msg = (String) EasyMock.getCurrentArguments()[0];

                debugMessages.add(msg);

                return null;
            }).anyTimes();

            log.info(EasyMock.anyObject(String.class));
            EasyMock.expectLastCall().andAnswer(() -> {

                String msg = (String) EasyMock.getCurrentArguments()[0];

                infoMessages.add(msg);

                return null;
            }).anyTimes();

            PowerMock.replay(log);

            return log;
        }

        protected Logger createMockLogger() {

            Logger log = PowerMock.createMock(Logger.class);

            if (null == debugMessages) {

                debugMessages = new ArrayList<>();
            }

            if (null == infoMessages) {

                infoMessages = new ArrayList<>();
            }

            log.debug(EasyMock.anyObject(String.class));
            EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

                @Override
                public Object answer() throws Throwable {

                    String msg = (String) EasyMock.getCurrentArguments()[0];

                    debugMessages.add(msg);

                    return null;
                }

            }).anyTimes();

            log.info(EasyMock.anyObject(String.class));
            EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

                @Override
                public Object answer() throws Throwable {

                    String msg = (String) EasyMock.getCurrentArguments()[0];

                    infoMessages.add(msg);

                    return null;
                }

            }).anyTimes();

            PowerMock.replay(log);

            return log;
        }

Replace Anonymous With Lambda datawave.ingest.table.config.AbstractTableConfigHelperTest.TestAbstractTableConfigHelperImpl.createMockLogger.anyTimes.andAnswer.new IAnswer<Object> with () -> {String msg=(String)EasyMock.getCurrentArguments()[0]; in method protected createMockLogger() : Logger from class datawave.ingest.table.config.AbstractTableConfigHelperTest.TestAbstractTableConfigHelperImpl


    @SuppressWarnings("unchecked")
    public TableOperations mockUpTableOperations() throws AccumuloException, TableNotFoundException, AccumuloSecurityException {

        tableProperties = new HashMap<>();
        for (int index = 0; index < AbstractTableConfigHelperTest.FIXED_KEYS.length; index++) {

            tableProperties.put(AbstractTableConfigHelperTest.FIXED_KEYS[index], AbstractTableConfigHelperTest.FIXED_VALUES[index]);
        }

        localityGroups = new HashMap<>();

        for (int index = 0; index < AbstractTableConfigHelperTest.FIXED_KEYS.length; index++) {
            Set<Text> values = new HashSet<>();

            localityGroups.put(AbstractTableConfigHelperTest.FIXED_KEYS[index], values);

            for (int counter = 0; counter < AbstractTableConfigHelperTest.FIXED_VALUES.length; counter++) {

                values.add(new Text(AbstractTableConfigHelperTest.FIXED_VALUES[counter]));
            }
        }

        TableOperations mock = PowerMock.createMock(TableOperations.class);

        mock.getProperties(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            AbstractTableConfigHelperTest.GET_PROPERTIES_CALLED = true;

            String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

            if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
            }

            if (AbstractTableConfigHelperTest.GET_PROPERTIES_THROWS_ACCUMULO_EXCEPTION) {

                throw new AccumuloException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
            }

            return tableProperties.entrySet();
        }).anyTimes();

        mock.setProperty(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            AbstractTableConfigHelperTest.SET_PROPERTIES_CALLED = true;

            String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

            if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
            }

            if (AbstractTableConfigHelperTest.SET_PROPERTIES_THROWS_ACCUMULO_SECUIRTY_EXCEPTION) {

                throw new AccumuloSecurityException(AbstractTableConfigHelperTest.USER, SecurityErrorCode.PERMISSION_DENIED);
            }

            String name = (String) EasyMock.getCurrentArguments()[1];
            String value = (String) EasyMock.getCurrentArguments()[2];

            tableProperties.put(name, value);
            return null;
        }).anyTimes();

        mock.getLocalityGroups(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            AbstractTableConfigHelperTest.ARE_LOCALITY_GROUPS_CONFIGURED_CALLED = true;

            String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

            if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
            }

            if (AbstractTableConfigHelperTest.ARE_LOCALITY_GROUPS_CONFIGURED_THROWS_ACCUMULO_SECUIRTY_EXCEPTION) {

                throw new AccumuloSecurityException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE, SecurityErrorCode.DEFAULT_SECURITY_ERROR);
            }

            return localityGroups;
        }).anyTimes();

        Map<String,Set<Text>> groups = new HashMap<>();

        mock.setLocalityGroups(EasyMock.anyObject(String.class), EasyMock.anyObject(groups.getClass()));
        EasyMock.expectLastCall().andAnswer(() -> {

            AbstractTableConfigHelperTest.SET_LOCALITY_GROUPS_CONFIGURED_CALLED = true;

            String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

            if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
            }

            if (AbstractTableConfigHelperTest.SET_LOCALITY_GROUPS_CONFIGURED_THROWS_ACCUMULO_SECUIRTY_EXCEPTION) {

                throw new AccumuloSecurityException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE, SecurityErrorCode.DEFAULT_SECURITY_ERROR);
            }

            if (AbstractTableConfigHelperTest.SET_LOCALITY_GROUPS_CONFIGURED_THROWS_ACCUMULO_EXCEPTION) {

                throw new AccumuloException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
            }

            Map<String,Set<Text>> groups1 = (Map<String,Set<Text>>) EasyMock.getCurrentArguments()[1];

            localityGroups.putAll(groups1);

            return null;
        }).anyTimes();

        // prepare it for use...
        PowerMock.replay(mock);

        return mock;
    }

    @SuppressWarnings("unchecked")
    public TableOperations mockUpTableOperations() throws AccumuloException, TableNotFoundException, AccumuloSecurityException {

        tableProperties = new HashMap<>();
        for (int index = 0; index < AbstractTableConfigHelperTest.FIXED_KEYS.length; index++) {

            tableProperties.put(AbstractTableConfigHelperTest.FIXED_KEYS[index], AbstractTableConfigHelperTest.FIXED_VALUES[index]);
        }

        localityGroups = new HashMap<>();

        for (int index = 0; index < AbstractTableConfigHelperTest.FIXED_KEYS.length; index++) {
            Set<Text> values = new HashSet<>();

            localityGroups.put(AbstractTableConfigHelperTest.FIXED_KEYS[index], values);

            for (int counter = 0; counter < AbstractTableConfigHelperTest.FIXED_VALUES.length; counter++) {

                values.add(new Text(AbstractTableConfigHelperTest.FIXED_VALUES[counter]));
            }
        }

        TableOperations mock = PowerMock.createMock(TableOperations.class);

        mock.getProperties(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            public Object answer() throws AccumuloException, TableNotFoundException {

                AbstractTableConfigHelperTest.GET_PROPERTIES_CALLED = true;

                String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

                if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                    throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
                }

                if (AbstractTableConfigHelperTest.GET_PROPERTIES_THROWS_ACCUMULO_EXCEPTION) {

                    throw new AccumuloException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
                }

                return tableProperties.entrySet();
            }

        }).anyTimes();

        mock.setProperty(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            public Object answer() throws TableNotFoundException, AccumuloSecurityException {

                AbstractTableConfigHelperTest.SET_PROPERTIES_CALLED = true;

                String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

                if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                    throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
                }

                if (AbstractTableConfigHelperTest.SET_PROPERTIES_THROWS_ACCUMULO_SECUIRTY_EXCEPTION) {

                    throw new AccumuloSecurityException(AbstractTableConfigHelperTest.USER, SecurityErrorCode.PERMISSION_DENIED);
                }

                String name = (String) EasyMock.getCurrentArguments()[1];
                String value = (String) EasyMock.getCurrentArguments()[2];

                tableProperties.put(name, value);
                return null;
            }

        }).anyTimes();

        mock.getLocalityGroups(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                AbstractTableConfigHelperTest.ARE_LOCALITY_GROUPS_CONFIGURED_CALLED = true;

                String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

                if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                    throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
                }

                if (AbstractTableConfigHelperTest.ARE_LOCALITY_GROUPS_CONFIGURED_THROWS_ACCUMULO_SECUIRTY_EXCEPTION) {

                    throw new AccumuloSecurityException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE, SecurityErrorCode.DEFAULT_SECURITY_ERROR);
                }

                return localityGroups;
            }

        }).anyTimes();

        Map<String,Set<Text>> groups = new HashMap<>();

        mock.setLocalityGroups(EasyMock.anyObject(String.class), EasyMock.anyObject(groups.getClass()));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                AbstractTableConfigHelperTest.SET_LOCALITY_GROUPS_CONFIGURED_CALLED = true;

                String tableNameParameter = (String) EasyMock.getCurrentArguments()[0];

                if (!AbstractTableConfigHelperTest.TABLE_NAME.equals(tableNameParameter)) {

                    throw new TableNotFoundException(null, tableNameParameter, AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
                }

                if (AbstractTableConfigHelperTest.SET_LOCALITY_GROUPS_CONFIGURED_THROWS_ACCUMULO_SECUIRTY_EXCEPTION) {

                    throw new AccumuloSecurityException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE, SecurityErrorCode.DEFAULT_SECURITY_ERROR);
                }

                if (AbstractTableConfigHelperTest.SET_LOCALITY_GROUPS_CONFIGURED_THROWS_ACCUMULO_EXCEPTION) {

                    throw new AccumuloException(AbstractTableConfigHelperTest.DEFAULT_EXCEPTION_MESSAGE);
                }

                Map<String,Set<Text>> groups = (Map<String,Set<Text>>) EasyMock.getCurrentArguments()[1];

                localityGroups.putAll(groups);

                return null;
            }

        }).anyTimes();

        // prepare it for use...
        PowerMock.replay(mock);

        return mock;
    }

Replace Anonymous With Lambda datawave.ingest.table.config.AbstractTableConfigHelperTest.mockUpTableOperations.anyTimes.andAnswer.new IAnswer<Object> with () -> {AbstractTableConfigHelperTest.GET_PROPERTIES_CALLED=true; in method public mockUpTableOperations() : TableOperations from class datawave.ingest.table.config.AbstractTableConfigHelperTest


    public Configuration createMockConfiguration() {

        if (null == configuration) {

            configuration = new HashMap<>();
        }

        Configuration mock = PowerMock.createMock(Configuration.class);

        mock.get(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            String results = null;
            String key = (String) EasyMock.getCurrentArguments()[0];
            String defaultValue = (String) EasyMock.getCurrentArguments()[1];

            if (configuration.containsKey(key)) {

                results = configuration.get(key);
            } else {

                results = defaultValue;
            }

            return results;
        }).anyTimes();

        mock.getBoolean(EasyMock.anyObject(String.class), EasyMock.anyBoolean());
        EasyMock.expectLastCall().andAnswer(() -> {

            String key = (String) EasyMock.getCurrentArguments()[0];
            boolean results = (Boolean) EasyMock.getCurrentArguments()[1];

            if (configuration.containsKey(key)) {

                results = Boolean.parseBoolean(configuration.get(key));
            }

            return results;
        }).anyTimes();

        PowerMock.replay(mock);

        return mock;
    }

    public Configuration createMockConfiguration() {

        if (null == configuration) {

            configuration = new HashMap<>();
        }

        Configuration mock = PowerMock.createMock(Configuration.class);

        mock.get(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                String results = null;
                String key = (String) EasyMock.getCurrentArguments()[0];
                String defaultValue = (String) EasyMock.getCurrentArguments()[1];

                if (configuration.containsKey(key)) {

                    results = configuration.get(key);
                } else {

                    results = defaultValue;
                }

                return results;
            }

        }).anyTimes();

        mock.getBoolean(EasyMock.anyObject(String.class), EasyMock.anyBoolean());
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                String key = (String) EasyMock.getCurrentArguments()[0];
                boolean results = (Boolean) EasyMock.getCurrentArguments()[1];

                if (configuration.containsKey(key)) {

                    results = Boolean.parseBoolean(configuration.get(key));
                }

                return results;
            }

        }).anyTimes();

        PowerMock.replay(mock);

        return mock;
    }

Replace Anonymous With Lambda datawave.ingest.table.config.DateIndexTableConfigHelperTest.createMockConfiguration.anyTimes.andAnswer.new IAnswer<Object> with () -> {String results=null; in method public createMockConfiguration() : Configuration from class datawave.ingest.table.config.DateIndexTableConfigHelperTest


    protected Logger createMockLogger() {

        Logger log = PowerMock.createMock(Logger.class);

        if (null == debugMessages) {

            debugMessages = new ArrayList<>();
        }

        if (null == infoMessages) {

            infoMessages = new ArrayList<>();
        }

        log.debug(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            String msg = (String) EasyMock.getCurrentArguments()[0];

            debugMessages.add(msg);

            return null;
        }).anyTimes();

        log.info(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(() -> {

            String msg = (String) EasyMock.getCurrentArguments()[0];

            infoMessages.add(msg);

            return null;
        }).anyTimes();

        PowerMock.replay(log);

        return log;
    }

    protected Logger createMockLogger() {

        Logger log = PowerMock.createMock(Logger.class);

        if (null == debugMessages) {

            debugMessages = new ArrayList<>();
        }

        if (null == infoMessages) {

            infoMessages = new ArrayList<>();
        }

        log.debug(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                String msg = (String) EasyMock.getCurrentArguments()[0];

                debugMessages.add(msg);

                return null;
            }

        }).anyTimes();

        log.info(EasyMock.anyObject(String.class));
        EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

            @Override
            public Object answer() throws Throwable {

                String msg = (String) EasyMock.getCurrentArguments()[0];

                infoMessages.add(msg);

                return null;
            }

        }).anyTimes();

        PowerMock.replay(log);

        return log;
    }

Replace Anonymous With Lambda datawave.ingest.table.config.DateIndexTableConfigHelperTest.createMockLogger.anyTimes.andAnswer.new IAnswer<Object> with () -> {String msg=(String)EasyMock.getCurrentArguments()[0]; in method protected createMockLogger() : Logger from class datawave.ingest.table.config.DateIndexTableConfigHelperTest


    @Test
    public void testDeleteFullMajcWithDeepCopyThreaded() throws IOException, InterruptedException, ExecutionException {
        TreeMultimap<Key,Value> map = TreeMultimap.create();

        map.put(newKey("20121002_1", "UUID", "abc", true), new Value(createNewUidList("abc.0").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.1").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.2").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.3").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.4").build().toByteArray()));

        map.put(newKey("20121002_1", "UUID", "abd"), new Value(createNewUidList("abc.3").build().toByteArray()));

        SortedMultiMapIterator data = new SortedMultiMapIterator(map);

        final PropogatingIterator iter = new PropogatingIterator();
        Map<String,String> options = Maps.newHashMap();

        options.put(PropogatingIterator.AGGREGATOR_DEFAULT, GlobalIndexUidAggregator.class.getCanonicalName());

        final IteratorEnvironment env = new MockIteratorEnvironment(true);

        iter.init(data, options, env);
        ExecutorService exec = Executors.newFixedThreadPool(10);

        List<Future<List<Entry<Key,Value>>>> futures = Lists.newArrayList();
        for (int i = 0; i < 25; i++) {
            futures.add(exec.submit(() -> {
                SortedKeyValueIterator<Key,Value> myitr = iter.deepCopy(env);
                Random rand = new Random();
                LockSupport.parkNanos(rand.nextInt(10));

                myitr.seek(new Range(), Collections.<ByteSequence> emptyList(), false);

                List<Entry<Key,Value>> resultList = Lists.newArrayList();

                Assert.assertTrue(myitr.hasTop());

                Key topKey = myitr.getTopKey();
                resultList.add(Maps.immutableEntry(topKey, myitr.getTopValue()));
                myitr.next();
                resultList.add(Maps.immutableEntry(myitr.getTopKey(), myitr.getTopValue()));

                return resultList;
            }));
        }

        for (Future<List<Entry<Key,Value>>> future : futures) {
            List<Entry<Key,Value>> kvList = future.get();

            Assert.assertEquals(2, kvList.size());

            Entry<Key,Value> kv1 = kvList.get(0);
            Entry<Key,Value> kv2 = kvList.get(1);

            Assert.assertEquals(newKey("20121002_1", "UUID", "abc", true), kv1.getKey());
            validateUids(kv1.getValue(), "abc.0", "abc.1", "abc.2", "abc.3", "abc.4");
            Assert.assertEquals(newKey("20121002_1", "UUID", "abd"), kv2.getKey());
        }
        exec.shutdownNow();

    }

    @Test
    public void testDeleteFullMajcWithDeepCopyThreaded() throws IOException, InterruptedException, ExecutionException {
        TreeMultimap<Key,Value> map = TreeMultimap.create();

        map.put(newKey("20121002_1", "UUID", "abc", true), new Value(createNewUidList("abc.0").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.1").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.2").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.3").build().toByteArray()));
        map.put(newKey("20121002_1", "UUID", "abc"), new Value(createNewUidList("abc.4").build().toByteArray()));

        map.put(newKey("20121002_1", "UUID", "abd"), new Value(createNewUidList("abc.3").build().toByteArray()));

        SortedMultiMapIterator data = new SortedMultiMapIterator(map);

        final PropogatingIterator iter = new PropogatingIterator();
        Map<String,String> options = Maps.newHashMap();

        options.put(PropogatingIterator.AGGREGATOR_DEFAULT, GlobalIndexUidAggregator.class.getCanonicalName());

        final IteratorEnvironment env = new MockIteratorEnvironment(true);

        iter.init(data, options, env);
        ExecutorService exec = Executors.newFixedThreadPool(10);

        List<Future<List<Entry<Key,Value>>>> futures = Lists.newArrayList();
        for (int i = 0; i < 25; i++) {
            futures.add(exec.submit(new Callable<List<Entry<Key,Value>>>() {

                @Override
                public List<Entry<Key,Value>> call() throws Exception {
                    SortedKeyValueIterator<Key,Value> myitr = iter.deepCopy(env);
                    Random rand = new Random();
                    LockSupport.parkNanos(rand.nextInt(10));

                    myitr.seek(new Range(), Collections.<ByteSequence> emptyList(), false);

                    List<Entry<Key,Value>> resultList = Lists.newArrayList();

                    Assert.assertTrue(myitr.hasTop());

                    Key topKey = myitr.getTopKey();
                    resultList.add(Maps.immutableEntry(topKey, myitr.getTopValue()));
                    myitr.next();
                    resultList.add(Maps.immutableEntry(myitr.getTopKey(), myitr.getTopValue()));

                    return resultList;
                }

            }));
        }

        for (Future<List<Entry<Key,Value>>> future : futures) {
            List<Entry<Key,Value>> kvList = future.get();

            Assert.assertEquals(2, kvList.size());

            Entry<Key,Value> kv1 = kvList.get(0);
            Entry<Key,Value> kv2 = kvList.get(1);

            Assert.assertEquals(newKey("20121002_1", "UUID", "abc", true), kv1.getKey());
            validateUids(kv1.getValue(), "abc.0", "abc.1", "abc.2", "abc.3", "abc.4");
            Assert.assertEquals(newKey("20121002_1", "UUID", "abd"), kv2.getKey());
        }
        exec.shutdownNow();

    }

Replace Anonymous With Lambda datawave.iterators.PropogatingIteratorTest.testDeleteFullMajcWithDeepCopyThreaded.add.submit.new Callable<List<Entry<Key,Value>>> with () -> {SortedKeyValueIterator<Key,Value> myitr=iter.deepCopy(env); in method public testDeleteFullMajcWithDeepCopyThreaded() : void from class datawave.iterators.PropogatingIteratorTest


    /**
     * Test with Slice distributor
     */
    @Test
    public void testDateFolderDistributor() throws Exception {
        File f = setUpFlagDir();
        // 5 days, 5 files each day, two folders in fmc = 50 flags
        createTestFiles(5, 5);
        createBogusTestFiles(5, 5);
        createCopyingTestFiles(5, 5);
        fmc.setDistributorType("folderdate");
        FlagMaker instance = new TestWrappedFlagMaker(fmc);
        instance.processFlags();
        File[] flags = f.listFiles(pathname -> pathname.toString().endsWith("flag"));
        assertEquals("Incorrect number of flags: " + Arrays.toString(flags), 5, flags.length);
        HashSet<Long> buckets = new HashSet<>();
        DateUtils du = new DateUtils();
        for (File file : flags) {
            BufferedReader r = new BufferedReader(new FileReader(file));
            String[] files = r.readLine().split(" ")[1].split(",");
            // 10 inputs per flag file
            assertEquals("Incorrect number of files: " + Arrays.toString(files), 10, files.length);
            Map<String,Integer> folderCounts = new HashMap<>();
            Map<String,Integer> dateCounts = new HashMap<>();
            for (String string : files) {
                assertTrue(string.startsWith("target/test/BulkIngest/flagged"));
                // pull off the sub folder
                String folder = string.substring(31, 34);
                folderCounts.put(folder, (folderCounts.containsKey(folder) ? folderCounts.get(folder) : 0) + 1);
                String date = string.substring(35, 45);
                dateCounts.put(date, (dateCounts.containsKey(date) ? dateCounts.get(date) : 0) + 1);
                buckets.add(du.getBucket("day", string));
            }
            assertEquals(2, folderCounts.size());
            for (String folder : folderCounts.keySet()) {
                assertEquals(5, folderCounts.get(folder).intValue());
            }
            assertEquals(5, dateCounts.size());
            for (String date : dateCounts.keySet()) {
                assertEquals(2, dateCounts.get(date).intValue());
            }
            // 2 from each folder for each day
            assertEquals("Incorrect number of buckets: " + buckets, 5, buckets.size());
            buckets.clear();
            r.close();
        }

    }

    /**
     * Test with Slice distributor
     */
    @Test
    public void testDateFolderDistributor() throws Exception {
        File f = setUpFlagDir();
        // 5 days, 5 files each day, two folders in fmc = 50 flags
        createTestFiles(5, 5);
        createBogusTestFiles(5, 5);
        createCopyingTestFiles(5, 5);
        fmc.setDistributorType("folderdate");
        FlagMaker instance = new TestWrappedFlagMaker(fmc);
        instance.processFlags();
        File[] flags = f.listFiles(new FileFilter() {
            @Override
            public boolean accept(File pathname) {
                return pathname.toString().endsWith("flag");
            }
        });
        assertEquals("Incorrect number of flags: " + Arrays.toString(flags), 5, flags.length);
        HashSet<Long> buckets = new HashSet<>();
        DateUtils du = new DateUtils();
        for (File file : flags) {
            BufferedReader r = new BufferedReader(new FileReader(file));
            String[] files = r.readLine().split(" ")[1].split(",");
            // 10 inputs per flag file
            assertEquals("Incorrect number of files: " + Arrays.toString(files), 10, files.length);
            Map<String,Integer> folderCounts = new HashMap<>();
            Map<String,Integer> dateCounts = new HashMap<>();
            for (String string : files) {
                assertTrue(string.startsWith("target/test/BulkIngest/flagged"));
                // pull off the sub folder
                String folder = string.substring(31, 34);
                folderCounts.put(folder, (folderCounts.containsKey(folder) ? folderCounts.get(folder) : 0) + 1);
                String date = string.substring(35, 45);
                dateCounts.put(date, (dateCounts.containsKey(date) ? dateCounts.get(date) : 0) + 1);
                buckets.add(du.getBucket("day", string));
            }
            assertEquals(2, folderCounts.size());
            for (String folder : folderCounts.keySet()) {
                assertEquals(5, folderCounts.get(folder).intValue());
            }
            assertEquals(5, dateCounts.size());
            for (String date : dateCounts.keySet()) {
                assertEquals(2, dateCounts.get(date).intValue());
            }
            // 2 from each folder for each day
            assertEquals("Incorrect number of buckets: " + buckets, 5, buckets.size());
            buckets.clear();
            r.close();
        }

    }

Replace Anonymous With Lambda datawave.util.flag.FlagMakerTest.testDateFolderDistributor.flags.listFiles.new FileFilter with pathname -> pathname.toString().endsWith("flag") in method public testDateFolderDistributor() : void from class datawave.util.flag.FlagMakerTest


    protected JsonObjectFlattenerImpl(Builder builder) {
        this.pathDelimiter = builder.pathDelimiter;
        this.mapKeyWhitelist = builder.fieldNameWhitelist != null ? new HashSet<>(builder.fieldNameWhitelist) : null;
        this.mapKeyBlacklist = builder.fieldNameBlacklist != null ? new HashSet<>(builder.fieldNameBlacklist) : null;
        this.flattenMode = builder.flattenMode;
        this.occurrenceDelimiter = builder.occurrenceDelimiter;

        // If a GROUPED* mode is enabled, then we force addArrayIndexToFieldName to false, since the additional
        // context is redundant in those cases. The property is really only relevant for NORMAL mode usage

        this.addArrayIndexToFieldName = (this.flattenMode == FlattenMode.GROUPED || this.flattenMode == FlattenMode.GROUPED_AND_NORMAL) ? false
                        : builder.addArrayIndexToFieldName;

        if (this.flattenMode == FlattenMode.GROUPED) {
            if (this.pathDelimiter.equals(this.occurrenceDelimiter)) {
                throw new IllegalStateException("path delimiter and occurrence delimiter cannot be the same");
            }
        }

        if (null == builder.nameNormalizer) {
            this.nameNormalizer = (elementName, parentKey) -> defaultNormalizeJsonElementName(elementName);
        } else {
            this.nameNormalizer = builder.nameNormalizer;
        }

        if (null == builder.keyValueNormalizer) {
            this.keyValueNormalizer = new MapKeyValueNormalizer() {
                @Override
                public String normalizeMapKey(String key, String value) throws IllegalStateException {
                    return defaultNormalizeMapKey(key, value);
                }

                @Override
                public String normalizeMapValue(String value, String key) throws IllegalStateException {
                    return defaultNormalizeMapValue(value, key);
                }
            };
        } else {
            this.keyValueNormalizer = builder.keyValueNormalizer;
        }

    }

    protected JsonObjectFlattenerImpl(Builder builder) {
        this.pathDelimiter = builder.pathDelimiter;
        this.mapKeyWhitelist = builder.fieldNameWhitelist != null ? new HashSet<>(builder.fieldNameWhitelist) : null;
        this.mapKeyBlacklist = builder.fieldNameBlacklist != null ? new HashSet<>(builder.fieldNameBlacklist) : null;
        this.flattenMode = builder.flattenMode;
        this.occurrenceDelimiter = builder.occurrenceDelimiter;

        // If a GROUPED* mode is enabled, then we force addArrayIndexToFieldName to false, since the additional
        // context is redundant in those cases. The property is really only relevant for NORMAL mode usage

        this.addArrayIndexToFieldName = (this.flattenMode == FlattenMode.GROUPED || this.flattenMode == FlattenMode.GROUPED_AND_NORMAL) ? false
                        : builder.addArrayIndexToFieldName;

        if (this.flattenMode == FlattenMode.GROUPED) {
            if (this.pathDelimiter.equals(this.occurrenceDelimiter)) {
                throw new IllegalStateException("path delimiter and occurrence delimiter cannot be the same");
            }
        }

        if (null == builder.nameNormalizer) {
            this.nameNormalizer = new JsonElementNameNormalizer() {
                @Override
                public String normalizeElementName(String elementName, String parentKey) throws IllegalStateException {
                    return defaultNormalizeJsonElementName(elementName);
                }
            };
        } else {
            this.nameNormalizer = builder.nameNormalizer;
        }

        if (null == builder.keyValueNormalizer) {
            this.keyValueNormalizer = new MapKeyValueNormalizer() {
                @Override
                public String normalizeMapKey(String key, String value) throws IllegalStateException {
                    return defaultNormalizeMapKey(key, value);
                }

                @Override
                public String normalizeMapValue(String value, String key) throws IllegalStateException {
                    return defaultNormalizeMapValue(value, key);
                }
            };
        } else {
            this.keyValueNormalizer = builder.keyValueNormalizer;
        }

    }

Replace Anonymous With Lambda datawave.ingest.json.util.JsonObjectFlattenerImpl.JsonObjectFlattenerImpl.new JsonElementNameNormalizer with (elementName, parentKey) -> defaultNormalizeJsonElementName(elementName) in method protected JsonObjectFlattenerImpl(builder Builder) from class datawave.ingest.json.util.JsonObjectFlattenerImpl