Input: Returns info about remote pipelines that have changed since the last sending of events [CODESPLIT] @ Override public List < PipelineAndValidationStatus > getRemotePipelinesWithChanges ( ) throws PipelineException { List < PipelineAndValidationStatus > pipelineAndValidationStatuses = new ArrayList <> ( ) ; for ( Pair < PipelineState , Map < String , String > > pipelineStateAndOffset : stateEventListener . getPipelineStateEvents ( ) ) { PipelineState pipelineState = pipelineStateAndOffset . getLeft ( ) ; Map < String , String > offset = pipelineStateAndOffset . getRight ( ) ; String name = pipelineState . getPipelineId ( ) ; String rev = pipelineState . getRev ( ) ; boolean isClusterMode = ( pipelineState . getExecutionMode ( ) != ExecutionMode . STANDALONE ) ? true : false ; List < WorkerInfo > workerInfos = new ArrayList <> ( ) ; String title ; int runnerCount = 0 ; if ( pipelineStore . hasPipeline ( name ) ) { title = pipelineStore . getInfo ( name ) . getTitle ( ) ; Runner runner = manager . getRunner ( name , rev ) ; if ( isClusterMode ) { workerInfos = getWorkers ( runner . getSlaveCallbackList ( CallbackObjectType . METRICS ) ) ; } runnerCount = runner . getRunnerCount ( ) ; } else { title = null ; } pipelineAndValidationStatuses . add ( new PipelineAndValidationStatus ( getSchGeneratedPipelineName ( name , rev ) , title , rev , pipelineState . getTimeStamp ( ) , true , pipelineState . getStatus ( ) , pipelineState . getMessage ( ) , workerInfos , isClusterMode , getSourceOffset ( name , offset ) , null , runnerCount ) ) ; } return pipelineAndValidationStatuses ; }


Input: Creates a future ack result from the given parameter . It is expected that the caller will eventually wait for the future ack result that is returned . [CODESPLIT] public static RemoteDataCollectorResult futureAck ( Future < AckEvent > futureResult ) { return new RemoteDataCollectorResult ( futureResult , null , false , null ) ; }


Input: Get an operation code from record . First look for sdc . operation . code from record header . If not set look for __$operation in record . It is a specific field that MS SQL CDC origin set . [CODESPLIT] @ Override @ VisibleForTesting int getOperationFromRecord ( Record record , int defaultOpCode , UnsupportedOperationAction unsupportedAction , List < OnRecordErrorException > errorRecords ) { int opCode = - 1 ; // -1 is invalid and not used in OperationType. String op = null ; try { // Try sdc.operation.type first op = record . getHeader ( ) . getAttribute ( OperationType . SDC_OPERATION_TYPE ) ; // If not set, look for oracle.cdc.operation in record header. if ( StringUtils . isBlank ( op ) ) { op = record . getHeader ( ) . getAttribute ( OracleCDCOperationCode . OPERATION ) ; if ( op != null ) { // Convert the Oracle specific operation code to SDC standard operation code opCode = OracleCDCOperationCode . convertFromOracleToSDCCode ( op ) ; } } else { opCode = JDBCOperationType . convertToIntCode ( op ) ; } if ( opCode == - 1 ) { opCode = defaultOpCode ; } } catch ( NumberFormatException | UnsupportedOperationException ex ) { LOG . debug ( "Operation obtained from record is not supported: {}. Handle by UnsupportedOpertaionAction {}. {}" , ex . getMessage ( ) , unsupportedAction . getLabel ( ) , ex ) ; switch ( unsupportedAction ) { case DISCARD : LOG . debug ( "Discarding record with unsupported operation {}" , op ) ; break ; case SEND_TO_ERROR : LOG . debug ( "Sending record to error due to unsupported operation {}" , op ) ; errorRecords . add ( new OnRecordErrorException ( record , JdbcErrors . JDBC_70 , op ) ) ; break ; case USE_DEFAULT : opCode = defaultOpCode ; break ; default : //unknown action LOG . debug ( "Sending record to error due to unknown operation: {}" , op ) ; } } return opCode ; }


Input: Kept for backward compatibility with runtime stats to be removed in future [CODESPLIT] public static Gauge < Map < String , Object > > createGauge ( MetricRegistry metrics , String name , Gauge gauge , final String pipelineName , final String pipelineRev ) { return create ( metrics , gauge , metricName ( name , GAUGE_SUFFIX ) , pipelineName , pipelineRev ) ; }


Input: Remove metric object ( regardless of it s type ) [CODESPLIT] private static boolean remove ( final MetricRegistry metrics , final String name , String pipelineName , String pipelineRev ) { final String jmxNamePrefix = jmxPipelinePrefix ( pipelineName , pipelineRev ) ; final MetricRegistry metricRegistry = sdcMetrics ; if ( metricRegistry != null ) { AccessController . doPrivileged ( new PrivilegedAction < Void > ( ) { @ Override public Void run ( ) { metricRegistry . remove ( jmxNamePrefix + name ) ; return null ; } } ) ; } return metrics . remove ( name ) ; }


Input: Records from MongoDB Oplog origin have a bit unique structure . [CODESPLIT] @ Override public SortedMap < String , String > getColumnsToParameters ( final Record record , int op , Map < String , String > parameters , Map < String , String > columnsToFields ) { SortedMap < String , String > columnsToParameters = new TreeMap <> ( ) ; for ( Map . Entry < String , String > entry : columnsToFields . entrySet ( ) ) { String columnName = entry . getKey ( ) ; String fieldPath = getFieldPath ( columnName , columnsToFields , op ) ; if ( record . has ( fieldPath ) ) { columnsToParameters . put ( columnName , parameters . get ( columnName ) ) ; } else { LOG . trace ( "Record is missing a field for column {} for the operation code {}" , columnName , op ) ; } } return columnsToParameters ; }


Input: Return a fieldpath for the corresponding column name . The columnToField Map contains column - to - field mapping so simply get fieldpath for the column name if operation is INSERT or DELETE . We replace / o with / o2 to get _id field and change / o to / o / $set to get updating column & value fields . [CODESPLIT] @ Override String getFieldPath ( String columnName , Map < String , String > columnsToField , int op ) { if ( op == OperationType . UPDATE_CODE ) { String fieldPath = columnsToField . get ( columnName ) ; if ( fieldPath == null ) { LOG . error ( "Column name {} is not defined in column-filed mapping" , columnName ) ; return null ; } if ( fieldPath . contains ( ID_FIELD ) ) { // _id is stored in "/o2/column_name for update records. Need to change the fieldpath" return fieldPath . replace ( OP_FIELD , OP2_FIELD ) ; } else { // column and values are stored in "/o/$set/column_name". Need to change the fieldpath return fieldPath . replaceFirst ( OP_FIELD , String . format ( "%s/%s" , OP_FIELD , SET_FIELD ) ) ; } } return columnsToField . get ( columnName ) ; }


Input: Generate the Header attributes [CODESPLIT] private Map < String , Object > generateHeaderAttrs ( Path file ) throws StageException { try { Map < String , Object > recordHeaderAttr = new HashMap <> ( ) ; recordHeaderAttr . put ( HeaderAttributeConstants . FILE , file . toAbsolutePath ( ) ) ; recordHeaderAttr . put ( HeaderAttributeConstants . FILE_NAME , file . getFileName ( ) ) ; recordHeaderAttr . put ( HeaderAttributeConstants . SIZE , Files . size ( file ) ) ; recordHeaderAttr . put ( HeaderAttributeConstants . LAST_MODIFIED_TIME , Files . getLastModifiedTime ( file ) ) ; return recordHeaderAttr ; } catch ( IOException e ) { throw new TransformerStageCheckedException ( Errors . CONVERT_09 , e . toString ( ) , e ) ; } }


Input: Validate the record is a whole file record [CODESPLIT] private void validateRecord ( Record record ) throws StageException { try { FileRefUtil . validateWholeFileRecord ( record ) ; } catch ( IllegalArgumentException e ) { throw new TransformerStageCheckedException ( Errors . CONVERT_01 , e . toString ( ) , e ) ; } }


Input: Return the temporary parquet file path and validate the path [CODESPLIT] @ VisibleForTesting Path getAndValidateTempFilePath ( Record record , String sourceFileName ) throws StageException { RecordEL . setRecordInContext ( variables , record ) ; String dirPath ; try { dirPath = resolveEL ( tempDirElEval , variables , jobConfig . tempDir , String . class ) ; } catch ( ELEvalException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_04 , jobConfig . tempDir ) ; } if ( Strings . isNullOrEmpty ( dirPath ) ) { throw new TransformerStageCheckedException ( Errors . CONVERT_02 , jobConfig . tempDir ) ; } if ( Strings . isNullOrEmpty ( sourceFileName ) ) { throw new TransformerStageCheckedException ( Errors . CONVERT_03 , FILENAME ) ; } String fileName = jobConfig . uniquePrefix + sourceFileName + jobConfig . fileNameSuffix ; Path tempParquetFile = Paths . get ( dirPath , fileName ) ; if ( ! tempParquetFile . isAbsolute ( ) ) { throw new TransformerStageCheckedException ( Errors . CONVERT_05 , tempParquetFile ) ; } try { if ( ! Files . exists ( tempParquetFile . getParent ( ) ) ) { Files . createDirectories ( Paths . get ( dirPath ) ) ; } } catch ( IOException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_10 , tempParquetFile . toString ( ) , ex ) ; } // handle old temp files try { handleOldTempFiles ( tempParquetFile ) ; } catch ( IOException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_06 , tempParquetFile . toString ( ) , ex ) ; } return tempParquetFile ; }


Input: Delete temporary parquet file [CODESPLIT] private void handleOldTempFiles ( Path tempParquetFile ) throws IOException { if ( tempParquetFile == null ) { LOG . warn ( "temporary parquet file is empty" ) ; return ; } Files . deleteIfExists ( tempParquetFile ) ; }


Input: Return the Avro file input stream [CODESPLIT] private InputStream getAvroInputStream ( Record record ) throws StageException { try { FileRef fileRef = record . get ( FileRefUtil . FILE_REF_FIELD_PATH ) . getValueAsFileRef ( ) ; // get avro reader final boolean includeChecksumInTheEvents = false ; InputStream is = FileRefUtil . getReadableStream ( getContext ( ) , fileRef , InputStream . class , includeChecksumInTheEvents , null , null ) ; return is ; } catch ( IOException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_07 , ex . toString ( ) , ex ) ; } }


Input: Return the Avro file reader [CODESPLIT] private DataFileStream < GenericRecord > getFileReader ( InputStream is , String sourceFileName ) throws StageException { try { DatumReader < GenericRecord > reader = new GenericDatumReader <> ( ) ; DataFileStream < GenericRecord > fileReader = new DataFileStream <> ( is , reader ) ; return fileReader ; } catch ( IOException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_11 , sourceFileName , ex ) ; } }


Input: Convert Avro record to Parquet [CODESPLIT] private void writeParquet ( String sourceFileName , DataFileStream < GenericRecord > fileReader , Path tempParquetFile ) throws StageException { long recordCount = 0 ; GenericRecord avroRecord ; Schema schema = fileReader . getSchema ( ) ; LOG . debug ( "Start reading input file : {}" , sourceFileName ) ; try { // initialize parquet writer Configuration jobConfiguration = new Configuration ( ) ; String compressionCodecName = compressionElEval . eval ( variables , jobConfig . avroParquetConfig . compressionCodec , String . class ) ; jobConfiguration . set ( AvroParquetConstants . COMPRESSION_CODEC_NAME , compressionCodecName ) ; jobConfiguration . setInt ( AvroParquetConstants . ROW_GROUP_SIZE , jobConfig . avroParquetConfig . rowGroupSize ) ; jobConfiguration . setInt ( AvroParquetConstants . PAGE_SIZE , jobConfig . avroParquetConfig . pageSize ) ; jobConfiguration . setInt ( AvroParquetConstants . DICTIONARY_PAGE_SIZE , jobConfig . avroParquetConfig . dictionaryPageSize ) ; jobConfiguration . setInt ( AvroParquetConstants . MAX_PADDING_SIZE , jobConfig . avroParquetConfig . maxPaddingSize ) ; // Parquet writer ParquetWriter . Builder builder = AvroToParquetConverterUtil . initializeWriter ( new org . apache . hadoop . fs . Path ( tempParquetFile . toString ( ) ) , schema , jobConfiguration ) ; parquetWriter = builder . build ( ) ; while ( fileReader . hasNext ( ) ) { avroRecord = fileReader . next ( ) ; parquetWriter . write ( avroRecord ) ; recordCount ++ ; } parquetWriter . close ( ) ; } catch ( IOException ex ) { throw new TransformerStageCheckedException ( Errors . CONVERT_08 , sourceFileName , recordCount , ex ) ; } LOG . debug ( "Finished writing {} records to {}" , recordCount , tempParquetFile . getFileName ( ) ) ; }


Input: { [CODESPLIT] @ Override public void destroy ( ) { // Clean up any open resources. if ( null != redisClient ) { redisClient . disconnect ( ) ; redisClient . close ( ) ; redisClient = null ; } super . destroy ( ) ; }


Input: Visible for testing only [CODESPLIT] void validateClass ( String name ) { // python scripting engine generates __*__ classes under all packages (including SDC api package) if ( ! ( name . endsWith ( "__" ) ) ) { for ( String blacklistedPackage : blacklistedPackages ) { if ( name . startsWith ( blacklistedPackage ) ) { throw new IllegalArgumentException ( String . format ( "Class '%s' cannot be present in %s" , name , toString ( ) ) ) ; } } } }


Input: Visible for testing only [CODESPLIT] void validateResource ( String name ) { for ( String blacklistedPackage : blacklistedDirs ) { if ( name . startsWith ( blacklistedPackage ) ) { throw new IllegalArgumentException ( String . format ( "Resource '%s' cannot be present in %s" , name , toString ( ) ) ) ; } } }


Input: Checks that the encryption input is a supported type otherwise sends the record to error . [CODESPLIT] public Optional < Field > checkInputEncrypt ( Record record , Field field ) { if ( UNSUPPORTED_TYPES . contains ( field . getType ( ) ) ) { getContext ( ) . toError ( record , CRYPTO_03 , field . getType ( ) ) ; return Optional . empty ( ) ; } return Optional . of ( field ) ; }


Input: Checks that the encryption input is a supported type otherwise sends the record to StageException . [CODESPLIT] public Optional < Field > checkInputEncrypt ( Field field ) throws StageException { if ( UNSUPPORTED_TYPES . contains ( field . getType ( ) ) ) { throw new StageException ( CRYPTO_03 , field . getType ( ) ) ; } return Optional . of ( field ) ; }


Input: Checks that the decryption input is a valid type otherwise sends the record to error . [CODESPLIT] public Optional < Field > checkInputDecrypt ( Record record , Field field ) { if ( field . getType ( ) != Field . Type . BYTE_ARRAY ) { getContext ( ) . toError ( record , CRYPTO_02 , field . getType ( ) ) ; return Optional . empty ( ) ; } return Optional . of ( field ) ; }


Input: Checks that the decryption input is a valid type otherwise sends the record to StageException . [CODESPLIT] public Optional < Field > checkInputDecrypt ( Field field ) throws StageException { if ( field . getType ( ) != Field . Type . BYTE_ARRAY ) { throw new StageException ( CRYPTO_02 , field . getType ( ) ) ; } return Optional . of ( field ) ; }


Input: Does data type conversions in preparation for encryption . [CODESPLIT] public byte [ ] prepareEncrypt ( Field field , Map < String , String > context ) { context . put ( SDC_FIELD_TYPE , field . getType ( ) . name ( ) ) ; if ( field . getType ( ) == Field . Type . BYTE_ARRAY ) { return field . getValueAsByteArray ( ) ; } else { // Treat all other data as strings return field . getValueAsString ( ) . getBytes ( Charsets . UTF_8 ) ; } }


Input: Returns a decrypted { @link Field } with its original type preserved when type information has been preserved in the AAD . [CODESPLIT] public Field createResultFieldDecrypt ( CryptoResult < byte [ ] , ? > result ) { Field . Type fieldType = Field . Type . valueOf ( result . getEncryptionContext ( ) . getOrDefault ( SDC_FIELD_TYPE , Field . Type . BYTE_ARRAY . name ( ) ) ) ; // Field API prohibits STRING to BYTE_ARRAY conversion so this is a special case if ( fieldType == Field . Type . BYTE_ARRAY ) { return Field . create ( result . getResult ( ) ) ; } // Field API supports STRING to other primitive types. return Field . create ( fieldType , new String ( result . getResult ( ) ) ) ; }


Input: Processes a batch from the specified file and offset up to a maximum batch size . If the file is fully processed it must return - 1 otherwise it must return the offset to continue from next invocation . [CODESPLIT] public String generateBatch ( WrappedFile file , String offset , int maxBatchSize , BatchMaker batchMaker ) throws StageException , BadSpoolFileException { if ( offset == null ) { offset = "0" ; } String sourceFile = file . getFileName ( ) ; try { if ( parser == null ) { parser = SpoolDirUtil . getParser ( fs , file , conf . dataFormat , parserFactory , offset , conf . dataFormatConfig . wholeFileMaxObjectLen , rateLimitElEval , rateLimitElVars , conf . dataFormatConfig . rateLimit ) ; } Map < String , Object > recordHeaderAttr = generateHeaderAttrs ( file ) ; for ( int i = 0 ; i < maxBatchSize ; i ++ ) { try { Record record ; try { record = parser . parse ( ) ; } catch ( RecoverableDataParserException ex ) { // Propagate partially parsed record to error stream record = ex . getUnparsedRecord ( ) ; recordHeaderAttr . put ( HeaderAttributeConstants . OFFSET , offset ) ; setHeaders ( record , recordHeaderAttr ) ; errorRecordHandler . onError ( new OnRecordErrorException ( record , ex . getErrorCode ( ) , ex . getParams ( ) ) ) ; perFileErrorCount ++ ; noMoreDataErrorCount ++ ; // We'll simply continue reading once this continue ; } if ( record != null ) { recordHeaderAttr . put ( HeaderAttributeConstants . OFFSET , offset ) ; setHeaders ( record , recordHeaderAttr ) ; batchMaker . addRecord ( record ) ; offset = parser . getOffset ( ) ; if ( offset == null ) { offset = "0" ; } noMoreDataRecordCount ++ ; perFileRecordCount ++ ; } else { parser . close ( ) ; parser = null ; offset = MINUS_ONE ; break ; } } catch ( ObjectLengthException ex ) { String exOffset = offset ; offset = ( parser != null ) ? parser . getOffset ( ) : MINUS_ONE ; if ( offset == null ) { offset = "0" ; } errorRecordHandler . onError ( Errors . SPOOLDIR_02 , sourceFile , exOffset , ex ) ; perFileErrorCount ++ ; noMoreDataErrorCount ++ ; } } } catch ( IOException | DataParserException ex ) { if ( ex instanceof ClosedByInterruptException || ex . getCause ( ) instanceof ClosedByInterruptException ) { //If the pipeline was stopped, we may get a ClosedByInterruptException while reading avro data. //This is because the thread is interrupted when the pipeline is stopped. //Instead of sending the file to error, publish batch and move one. } else { offset = MINUS_ONE ; String exOffset ; if ( ex instanceof OverrunException ) { exOffset = String . valueOf ( ( ( OverrunException ) ex ) . getStreamOffset ( ) ) ; } else { try { exOffset = ( parser != null ) ? parser . getOffset ( ) : MINUS_ONE ; } catch ( IOException ex1 ) { LOG . warn ( "Could not get the file offset to report with error, reason: {}" , ex1 . toString ( ) , ex ) ; exOffset = MINUS_ONE ; } } switch ( context . getOnErrorRecord ( ) ) { case DISCARD : break ; case TO_ERROR : // we failed to produce a record, which leaves the input file in an unknown state. all we can do here is // throw an exception. throw new BadSpoolFileException ( file . getAbsolutePath ( ) , exOffset , ex ) ; case STOP_PIPELINE : context . reportError ( Errors . SPOOLDIR_04 , sourceFile , exOffset , ex . toString ( ) , ex ) ; throw new StageException ( Errors . SPOOLDIR_04 , sourceFile , exOffset , ex . toString ( ) ) ; default : throw new IllegalStateException ( Utils . format ( "Unknown OnError value '{}'" , context . getOnErrorRecord ( ) , ex ) ) ; } } } finally { if ( MINUS_ONE . equals ( offset ) ) { if ( parser != null ) { try { parser . close ( ) ; parser = null ; } catch ( IOException ex ) { //NOP } } } } return offset ; }


Input: Initialize the gauge with needed information [CODESPLIT] private void initGaugeIfNeeded ( ) { gaugeMap . put ( THREAD_NAME , Thread . currentThread ( ) . getName ( ) ) ; gaugeMap . put ( STATUS , "" ) ; gaugeMap . put ( CURRENT_FILE , "" ) ; }


Input: Handle Exception [CODESPLIT] private void handleStageError ( ErrorCode errorCode , Exception e ) { final String errorMessage = "Failure Happened" ; LOG . error ( errorMessage , e ) ; try { errorRecordHandler . onError ( errorCode , e ) ; } catch ( StageException se ) { LOG . error ( "Error when routing to stage error" , se ) ; //Way to throw stage exception from runnable to main source thread Throwables . propagate ( se ) ; } }


Input: Find cycles in the { [CODESPLIT] boolean isGraphCyclic ( ) { for ( V vertex : directedGraph . vertices ( ) ) { boolean areThereCycles = findCycles ( new LinkedHashSet < V > ( ) , vertex ) ; if ( areThereCycles ) { return true ; } } return false ; }


Input: Changes the Pattern that this Matcher uses to find matches with [CODESPLIT] public Matcher usePattern ( Pattern newPattern ) { if ( newPattern == null ) { throw new IllegalArgumentException ( "newPattern cannot be null" ) ; } this . parentPattern = newPattern ; matcher . usePattern ( newPattern . pattern ( ) ) ; return this ; }


Input: Implements a non - terminal append - and - replace step . [CODESPLIT] public Matcher appendReplacement ( StringBuffer sb , String replacement ) { matcher . appendReplacement ( sb , parentPattern . replaceProperties ( replacement ) ) ; return this ; }


Input: Finds all named groups that exist in the input string . This resets the matcher and attempts to match the input against the pre - specified pattern . [CODESPLIT] @ Override public Map < String , String > namedGroups ( ) { Map < String , String > result = new LinkedHashMap < String , String > ( ) ; if ( matcher . find ( 0 ) ) { for ( String groupName : parentPattern . groupNames ( ) ) { String groupValue = matcher . group ( groupIndex ( groupName ) ) ; result . put ( groupName , groupValue ) ; } } return result ; }


Input: Replaces every subsequence of the input sequence that matches the pattern with the given replacement string . [CODESPLIT] public String replaceAll ( String replacement ) { String r = parentPattern . replaceProperties ( replacement ) ; return matcher . replaceAll ( r ) ; }


Input: no trespassing ... [CODESPLIT] private Properties getKafkaProperties ( ) { Properties props = new Properties ( ) ; props . putAll ( conf . streamsOptions ) ; props . setProperty ( "group.id" , conf . consumerGroup ) ; props . setProperty ( "max.poll.records" , String . valueOf ( batchSize ) ) ; props . setProperty ( "enable.auto.commit" , "true" ) ; props . setProperty ( "auto.commit.interval.ms" , "1000" ) ; props . setProperty ( "key.deserializer" , "org.apache.kafka.common.serialization.ByteArrayDeserializer" ) ; props . setProperty ( "value.deserializer" , "com.mapr.db.cdc.ChangeDataRecordDeserializer" ) ; return props ; }


Input: This method is used during deserializer and sqpath ( Single quote escaped path ) is passed to determine the last field name . [CODESPLIT] public static String getLastFieldNameFromPath ( String path ) { String [ ] pathSplit = ( path != null ) ? path . split ( "/" ) : null ; if ( pathSplit != null && pathSplit . length > 0 ) { String lastFieldName = pathSplit [ pathSplit . length - 1 ] ; //handle special case field name containing slash eg. /'foo/bar' boolean singleQuoted = lastFieldName . charAt ( 0 ) == QUOTE_CHAR && lastFieldName . charAt ( lastFieldName . length ( ) - 1 ) == QUOTE_CHAR ; if ( lastFieldName . contains ( "'" ) && ! singleQuoted ) { //If path contains slash inside name, split it by "/'" pathSplit = path . split ( "/'" ) ; if ( pathSplit . length > 0 ) { lastFieldName = "'" + pathSplit [ pathSplit . length - 1 ] ; singleQuoted = true ; } } if ( singleQuoted ) { return singleQuoteUnescape ( lastFieldName ) ; } else { return lastFieldName ; } } return path ; }


Input: This method escapes backslash double quotes and single quotes ( keeping replacement of to \\\\\ as is so as to maintain backward compatibility any serialization / deserialization ) [CODESPLIT] private static String escapeQuotesAndBackSlash ( String path , boolean isSingleQuoteEscape ) { String quoteChar = isSingleQuoteEscape ? "'" : "\"" ; StringBuilder sb = new StringBuilder ( path . length ( ) * 2 ) . append ( quoteChar ) ; char [ ] chars = path . toCharArray ( ) ; for ( char c : chars ) { if ( c == ' ' ) { sb . append ( "\\\\" ) ; } else if ( c == ' ' ) { sb . append ( isSingleQuoteEscape ? "\\\"" : "\\\\\"" ) ; } else if ( c == ' ' ) { sb . append ( isSingleQuoteEscape ? "\\\\\'" : "\\\'" ) ; } else { sb . append ( c ) ; } } return sb . append ( quoteChar ) . toString ( ) ; }


Input: This method un escapes backslash double quotes and single quotes ( keeping replacement of \\\\\ to as is so as to maintain backward compatibility any serialization / deserialization ) [CODESPLIT] private static String unescapeQuotesAndBackSlash ( String path , boolean isSingleQuoteUnescape ) { path = ( isSingleQuoteUnescape ) ? path . replace ( "\\\"" , "\"" ) . replace ( "\\\\\'" , "'" ) : path . replace ( "\\\\\"" , "\"" ) . replace ( "\\\'" , "'" ) ; return path . replace ( "\\\\" , "\\" ) ; }


Input: This method un escapes backslash and un escapes extra escapes before double quotes and single quotes ( appended by { [CODESPLIT] public static String standardizePathForParse ( String path , boolean isSingleQuoteEscape ) { path = isSingleQuoteEscape ? path . replace ( "\\\\\'" , "\\'" ) : path . replace ( "\\\\\"" , "\\\"" ) ; return path . replace ( "\\\\" , "\\" ) ; }


Input: There is a problem with some older stages that originally were not using our data parser library as the upgrade on those stages does not create all the properties that were introduced by the data parser library . For example File Tail origin doesn t support avro so the upgrade procedure doesn t use create the avro specific properties . This is normally not a problem as post - upgrade SDC will create all missing properties with default values . However this particular upgrade will fail if the property avroSchema is missing . [CODESPLIT] public static void ensureAvroSchemaExists ( List < Config > configs , String prefix ) { Optional < Config > avroSchema = findByName ( configs , "avroSchema" ) ; if ( ! avroSchema . isPresent ( ) ) { configs . add ( new Config ( prefix + ".avroSchema" , null ) ) ; } }


Input: Get global variable value . [CODESPLIT] public static String getGlobalVariable ( DataSource dataSource , String variable ) throws SQLException { try ( Connection conn = dataSource . getConnection ( ) ) { try ( Statement stmt = conn . createStatement ( ) ; ResultSet rs = stmt . executeQuery ( String . format ( "show global variables like '%s'" , variable ) ) ; ) { if ( rs . next ( ) ) { return rs . getString ( 2 ) ; } else { return "" ; } } } }


Input: ************************************************** [CODESPLIT] public static StatusJson wrapState ( PipelineStatus status ) { if ( status == null ) { return null ; } switch ( status ) { case STOPPED : return StatusJson . STOPPED ; case STOPPING : return StatusJson . STOPPING ; case RUNNING : return StatusJson . RUNNING ; case RUN_ERROR : return StatusJson . RUN_ERROR ; case FINISHED : return StatusJson . FINISHED ; case CONNECTING : return StatusJson . CONNECTING ; case CONNECT_ERROR : return StatusJson . CONNECT_ERROR ; case DISCONNECTED : return StatusJson . DISCONNECTED ; case DISCONNECTING : return StatusJson . DISCONNECTING ; case EDITED : return StatusJson . EDITED ; case FINISHING : return StatusJson . FINISHING ; case KILLED : return StatusJson . KILLED ; case RUNNING_ERROR : return StatusJson . RUNNING_ERROR ; case STARTING : return StatusJson . STARTING ; case STARTING_ERROR : return StatusJson . STARTING_ERROR ; case START_ERROR : return StatusJson . START_ERROR ; case RETRY : return StatusJson . RETRY ; case STOP_ERROR : return StatusJson . STOP_ERROR ; case STOPPING_ERROR : return StatusJson . STOPPING_ERROR ; case DELETED : return StatusJson . DELETED ; default : throw new IllegalArgumentException ( "Unrecognized state" + status ) ; } }


Input: Create pipeline start event . [CODESPLIT] private Record createStartEvent ( ) { Preconditions . checkState ( startEventStage != null , "Start Event Stage is not set!" ) ; EventRecord eventRecord = new EventRecordImpl ( "pipeline-start" , 1 , startEventStage . getInfo ( ) . getInstanceName ( ) , "" , null , null ) ; Map < String , Field > rootField = new LinkedHashMap <> ( ) ; rootField . put ( "user" , Field . create ( Field . Type . STRING , userContext . getUser ( ) ) ) ; rootField . put ( "pipelineId" , Field . create ( Field . Type . STRING , name ) ) ; rootField . put ( "pipelineTitle" , Field . create ( Field . Type . STRING , pipelineConf . getTitle ( ) ) ) ; // Pipeline parameters Map < String , Field > parameters = new LinkedHashMap <> ( ) ; if ( runtimeParameters != null ) { for ( Map . Entry < String , Object > entry : runtimeParameters . entrySet ( ) ) { parameters . put ( entry . getKey ( ) , Field . create ( Field . Type . STRING , entry . getValue ( ) . toString ( ) ) ) ; } } rootField . put ( "parameters" , Field . create ( parameters ) ) ; eventRecord . set ( Field . create ( rootField ) ) ; return eventRecord ; }


Input: Create pipeline stop event . [CODESPLIT] private Record createStopEvent ( PipelineStopReason stopReason ) { Preconditions . checkState ( stopEventStage != null , "Stop Event Stage is not set!" ) ; EventRecord eventRecord = new EventRecordImpl ( "pipeline-stop" , 1 , stopEventStage . getInfo ( ) . getInstanceName ( ) , "" , null , null ) ; Map < String , Field > rootField = new LinkedHashMap <> ( ) ; rootField . put ( "reason" , Field . create ( Field . Type . STRING , stopReason . name ( ) ) ) ; rootField . put ( "pipelineId" , Field . create ( Field . Type . STRING , name ) ) ; rootField . put ( "pipelineTitle" , Field . create ( Field . Type . STRING , pipelineConf . getTitle ( ) ) ) ; eventRecord . set ( Field . create ( rootField ) ) ; return eventRecord ; }


Input: infinite recursion [CODESPLIT] private void getAllReferences ( PartnerConnection partnerConnection , Map < String , ObjectMetadata > metadataMap , List < List < Pair < String , String > > > references , String [ ] allTypes , int depth ) throws ConnectionException { if ( depth < 0 ) { return ; } List < String > next = new ArrayList <> ( ) ; for ( int typeIndex = 0 ; typeIndex < allTypes . length ; typeIndex += MAX_METADATA_TYPES ) { int copyTo = Math . min ( typeIndex + MAX_METADATA_TYPES , allTypes . length ) ; String [ ] types = Arrays . copyOfRange ( allTypes , typeIndex , copyTo ) ; // Special case - we prepopulate the cache with the root sobject type - don't repeat // ourselves if ( types . length > 1 || ! metadataMap . containsKey ( types [ 0 ] ) ) { for ( DescribeSObjectResult result : partnerConnection . describeSObjects ( types ) ) { Map < String , Field > fieldMap = new LinkedHashMap <> ( ) ; Map < String , Field > relationshipMap = new LinkedHashMap <> ( ) ; for ( Field field : result . getFields ( ) ) { fieldMap . put ( field . getName ( ) . toLowerCase ( ) , field ) ; String relationshipName = field . getRelationshipName ( ) ; if ( relationshipName != null ) { relationshipMap . put ( relationshipName . toLowerCase ( ) , field ) ; } } Map < String , String > childRelationships = new LinkedHashMap <> ( ) ; for ( ChildRelationship child : result . getChildRelationships ( ) ) { if ( child . getRelationshipName ( ) != null ) { childRelationships . put ( child . getRelationshipName ( ) . toLowerCase ( ) , child . getChildSObject ( ) . toLowerCase ( ) ) ; } } metadataMap . put ( result . getName ( ) . toLowerCase ( ) , new ObjectMetadata ( fieldMap , relationshipMap , childRelationships ) ) ; } } if ( references != null ) { for ( List < Pair < String , String > > path : references ) { // Top field name in the path should be in the metadata now if ( ! path . isEmpty ( ) ) { Pair < String , String > top = path . get ( 0 ) ; Field field = metadataMap . get ( top . getLeft ( ) ) . getFieldFromRelationship ( top . getRight ( ) ) ; Set < String > sobjectNames = metadataMap . keySet ( ) ; for ( String ref : field . getReferenceTo ( ) ) { ref = ref . toLowerCase ( ) ; if ( ! sobjectNames . contains ( ref ) && ! next . contains ( ref ) ) { next . add ( ref ) ; } if ( path . size ( ) > 1 ) { path . set ( 1 , Pair . of ( ref , path . get ( 1 ) . getRight ( ) ) ) ; } } // SDC-10422 Polymorphic references have an implicit reference to the Name object type if ( field . isPolymorphicForeignKey ( ) ) { next . add ( NAME ) ; } path . remove ( 0 ) ; } } } } if ( ! next . isEmpty ( ) ) { getAllReferences ( partnerConnection , metadataMap , references , next . toArray ( new String [ 0 ] ) , depth - 1 ) ; } }


Input: since Salesforce doesn t like scientific notation in queries [CODESPLIT] protected String fixOffset ( String offsetColumn , String offset ) { com . sforce . soap . partner . Field sfdcField = getFieldMetadata ( sobjectType , offsetColumn ) ; if ( SobjectRecordCreator . DECIMAL_TYPES . contains ( sfdcField . getType ( ) . toString ( ) ) && offset . contains ( "E" ) ) { BigDecimal val = new BigDecimal ( offset ) ; offset = val . toPlainString ( ) ; if ( val . compareTo ( MAX_OFFSET_INT ) > 0 && ! offset . contains ( "." ) ) { // We need the ".0" suffix since Salesforce doesn't like integer // bigger than 2147483647 offset += ".0" ; } } return offset ; }


Input: RUNTIME supports only Numeric types and String at the moment [CODESPLIT] private Object extractAsRuntime ( Field field , String valueStr ) { if ( field . getType ( ) == Byte . TYPE || field . getType ( ) == Byte . class || field . getType ( ) == Short . TYPE || field . getType ( ) == Short . class || field . getType ( ) == Integer . TYPE || field . getType ( ) == Integer . class || field . getType ( ) == Long . TYPE || field . getType ( ) == Long . class || field . getType ( ) == Float . TYPE || field . getType ( ) == Float . class || field . getType ( ) == Double . TYPE || field . getType ( ) == Double . class ) { return extractAsNumber ( field , valueStr ) ; } else if ( String . class . isAssignableFrom ( field . getType ( ) ) ) { return valueStr ; } throw new IllegalArgumentException ( Utils . format ( "Invalid type for RUNTIME type: {}" , field . getType ( ) ) ) ; }


Input: Add a new partition to the given table with optional custom location . [CODESPLIT] public void executeAlterTableAddPartitionQuery ( String qualifiedTableName , LinkedHashMap < String , String > partitionNameValueMap , Map < String , HiveTypeInfo > partitionTypeMap , String partitionPath ) throws StageException { String sql = buildPartitionAdditionQuery ( qualifiedTableName , partitionNameValueMap , partitionTypeMap , partitionPath ) ; execute ( sql ) ; }


Input: Execute Alter Table set Table Properties [CODESPLIT] public void executeAlterTableSetTblPropertiesQuery ( String qualifiedTableName , String partitionPath ) throws StageException { String sql = buildSetTablePropertiesQuery ( qualifiedTableName , partitionPath ) ; execute ( sql ) ; }


Input: Returns { [CODESPLIT] public Set < PartitionInfoCacheSupport . PartitionValues > executeShowPartitionsQuery ( String qualifiedTableName ) throws StageException { String sql = buildShowPartitionsQuery ( qualifiedTableName ) ; return executeQuery ( sql , new WithResultSet < Set < PartitionInfoCacheSupport . PartitionValues > > ( ) { @ Override public Set < PartitionInfoCacheSupport . PartitionValues > run ( ResultSet rs ) throws SQLException , StageException { Set < PartitionInfoCacheSupport . PartitionValues > partitionValuesSet = new HashSet <> ( ) ; while ( rs . next ( ) ) { String partitionInfoString = rs . getString ( 1 ) ; String [ ] partitionInfoSplit = partitionInfoString . split ( HiveMetastoreUtil . SEP ) ; LinkedHashMap < String , String > vals = new LinkedHashMap <> ( ) ; for ( String partitionValInfo : partitionInfoSplit ) { String [ ] partitionNameVal = partitionValInfo . split ( "=" ) ; vals . put ( partitionNameVal [ 0 ] , partitionNameVal [ 1 ] ) ; } partitionValuesSet . add ( new PartitionInfoCacheSupport . PartitionValues ( vals ) ) ; } return partitionValuesSet ; } } ) ; }


Input: Returns { [CODESPLIT] public Pair < LinkedHashMap < String , HiveTypeInfo > , LinkedHashMap < String , HiveTypeInfo > > executeDescTableQuery ( String qualifiedTableName ) throws StageException { String sql = buildDescTableQuery ( qualifiedTableName ) ; return executeQuery ( sql , new WithResultSet < Pair < LinkedHashMap < String , HiveTypeInfo > , LinkedHashMap < String , HiveTypeInfo > > > ( ) { @ Override public Pair < LinkedHashMap < String , HiveTypeInfo > , LinkedHashMap < String , HiveTypeInfo > > run ( ResultSet rs ) throws SQLException , StageException { LinkedHashMap < String , HiveTypeInfo > columnTypeInfo = extractTypeInfo ( rs ) ; processDelimiter ( rs , "#" ) ; processDelimiter ( rs , "#" ) ; processDelimiter ( rs , "" ) ; LinkedHashMap < String , HiveTypeInfo > partitionTypeInfo = extractTypeInfo ( rs ) ; //Remove partition columns from the columns map. for ( String partitionCol : partitionTypeInfo . keySet ( ) ) { columnTypeInfo . remove ( partitionCol ) ; } return Pair . of ( columnTypeInfo , partitionTypeInfo ) ; } } ) ; }


Input: Returns location for given database . [CODESPLIT] public String executeDescribeDatabase ( String dbName ) throws StageException { String sql = buildDescribeDatabase ( dbName ) ; return executeQuery ( sql , rs -> { if ( ! rs . next ( ) ) { throw new HiveStageCheckedException ( Errors . HIVE_35 , "Database doesn't exists." ) ; } return HiveMetastoreUtil . stripHdfsHostAndPort ( rs . getString ( RESULT_SET_LOCATION ) ) ; } ) ; }


Input: Returns { [CODESPLIT] public Pair < Boolean , Boolean > executeShowTBLPropertiesQuery ( String qualifiedTableName ) throws StageException { String sql = String . format ( SHOW_TBLPROPERTIES , qualifiedTableName ) ; return executeQuery ( sql , new WithResultSet < Pair < Boolean , Boolean > > ( ) { @ Override public Pair < Boolean , Boolean > run ( ResultSet rs ) throws SQLException { boolean isExternal = false , useAsAvro = true ; while ( rs . next ( ) ) { String propName = rs . getString ( RESULT_SET_PROP_NAME ) ; String propValue = rs . getString ( RESULT_SET_PROP_VALUE ) ; if ( propName . toUpperCase ( ) . equals ( EXTERNAL ) ) { isExternal = Boolean . valueOf ( propValue ) ; } else if ( propName . equals ( AVRO_SCHEMA_URL ) ) { useAsAvro = false ; } } return Pair . of ( isExternal , useAsAvro ) ; } } ) ; }


Input: Execute given query . [CODESPLIT] private void execute ( String query ) throws StageException { LOG . debug ( "Executing SQL: {}" , query ) ; Timer . Context t = updateTimer . time ( ) ; try ( Statement statement = hiveConfigBean . getHiveConnection ( ) . createStatement ( ) ) { statement . execute ( query ) ; } catch ( Exception e ) { LOG . error ( "Exception while processing query: {}" , query , e ) ; throw new HiveStageCheckedException ( Errors . HIVE_20 , query , e . getMessage ( ) ) ; } finally { long time = t . stop ( ) ; LOG . debug ( "Query '{}' took {} nanoseconds" , query , time ) ; updateMeter . mark ( ) ; } }


Input: Execute given query and process it s result set . [CODESPLIT] private < T > T executeQuery ( String query , WithResultSet < T > execution ) throws StageException { LOG . debug ( "Executing SQL:  {}" , query ) ; Timer . Context t = selectTimer . time ( ) ; try ( Statement statement = hiveConfigBean . getHiveConnection ( ) . createStatement ( ) ; ResultSet rs = statement . executeQuery ( query ) ; ) { // Stop timer immediately so that we're calculating only query execution time and not the processing time long time = t . stop ( ) ; LOG . debug ( "Query '{}' took {} nanoseconds" , query , time ) ; t = null ; return execution . run ( rs ) ; } catch ( Exception e ) { LOG . error ( "Exception while processing query: {}" , query , e ) ; throw new HiveStageCheckedException ( Errors . HIVE_20 , query , e . getMessage ( ) ) ; } finally { // If the timer wasn't stopped due to exception yet, stop it now if ( t != null ) { long time = t . stop ( ) ; LOG . debug ( "Query '{}' took {} nanoseconds" , query , time ) ; } selectMeter . mark ( ) ; } }


Input: { [CODESPLIT] @ Override public String produce ( String lastSourceOffset , int maxBatchSize , BatchMaker batchMaker ) throws StageException { // Offsets can vary depending on the data source. Here we use an integer as an example only. long nextSourceOffset = 0 ; if ( lastSourceOffset != null ) { nextSourceOffset = Long . parseLong ( lastSourceOffset ) ; } int numRecords = 0 ; // TODO: As the developer, implement your logic that reads from a data source in this method. // Create records and add to batch. Records must have a string id. This can include the source offset // or other metadata to help uniquely identify the record itself. while ( numRecords < maxBatchSize ) { Record record = getContext ( ) . createRecord ( "some-id::" + nextSourceOffset ) ; Map < String , Field > map = new HashMap <> ( ) ; map . put ( "fieldName" , Field . create ( "Some Value" ) ) ; record . set ( Field . create ( map ) ) ; batchMaker . addRecord ( record ) ; ++ nextSourceOffset ; ++ numRecords ; } return String . valueOf ( nextSourceOffset ) ; }


Input: Run batch with given consumer for each pipe . [CODESPLIT] public void executeBatch ( String offsetKey , String offsetValue , long batchStartTime , ThrowingConsumer < Pipe > consumer ) throws PipelineRuntimeException , StageException { MDC . put ( LogConstants . RUNNER , String . valueOf ( runnerId ) ) ; // Persist static information for the batch (this won't change as the batch progresses) this . runtimeMetricGauge . put ( METRIC_BATCH_START_TIME , batchStartTime ) ; this . runtimeMetricGauge . put ( METRIC_OFFSET_KEY , Optional . ofNullable ( offsetKey ) . orElse ( "" ) ) ; this . runtimeMetricGauge . put ( METRIC_OFFSET_VALUE , Optional . ofNullable ( offsetValue ) . orElse ( "" ) ) ; this . runtimeMetricGauge . put ( METRIC_STAGE_START_TIME , System . currentTimeMillis ( ) ) ; try { // Run one pipe at a time for ( Pipe p : pipes ) { String instanceName = p . getStage ( ) . getInfo ( ) . getInstanceName ( ) ; this . runtimeMetricGauge . put ( METRIC_CURRENT_STAGE , instanceName ) ; MDC . put ( LogConstants . STAGE , instanceName ) ; if ( p instanceof StagePipe ) { this . runtimeMetricGauge . put ( METRIC_STAGE_START_TIME , System . currentTimeMillis ( ) ) ; } acceptConsumer ( consumer , p ) ; } // We've successfully finished batch this . runtimeMetricGauge . computeIfPresent ( METRIC_BATCH_COUNT , ( key , value ) -> ( ( long ) value ) + 1 ) ; } finally { resetBatchSpecificMetrics ( ) ; MDC . put ( LogConstants . RUNNER , "" ) ; MDC . put ( LogConstants . STAGE , "" ) ; } }


Input: Execute given consumer for each pipe rethrowing usual exceptions as RuntimeException . [CODESPLIT] public void forEach ( ThrowingConsumer < Pipe > consumer ) { try { MDC . put ( LogConstants . RUNNER , String . valueOf ( runnerId ) ) ; try { for ( Pipe p : pipes ) { MDC . put ( LogConstants . STAGE , p . getStage ( ) . getInfo ( ) . getInstanceName ( ) ) ; acceptConsumer ( consumer , p ) ; } } finally { MDC . put ( LogConstants . RUNNER , "" ) ; MDC . put ( LogConstants . STAGE , "" ) ; } } catch ( PipelineException | StageException e ) { throw new RuntimeException ( e ) ; } }


Input: Retrieve OffsetCommitTrigger pipe . [CODESPLIT] public OffsetCommitTrigger getOffsetCommitTrigger ( ) { for ( Pipe pipe : pipes ) { Stage stage = pipe . getStage ( ) . getStage ( ) ; if ( stage instanceof Target && stage instanceof OffsetCommitTrigger ) { return ( OffsetCommitTrigger ) stage ; } } return null ; }


Input: Return true if at least one stage is configured with STOP_PIPELINE for OnRecordError policy . [CODESPLIT] public boolean onRecordErrorStopPipeline ( ) { for ( Pipe pipe : pipes ) { StageContext stageContext = pipe . getStage ( ) . getContext ( ) ; if ( stageContext . getOnErrorRecord ( ) == OnRecordError . STOP_PIPELINE ) { return true ; } } return false ; }


Input: Accept given consumer and proper log context of any exception . [CODESPLIT] private void acceptConsumer ( ThrowingConsumer < Pipe > consumer , Pipe p ) throws PipelineRuntimeException , StageException { try { // Process pipe consumer . accept ( p ) ; } catch ( Throwable t ) { String instanceName = p . getStage ( ) . getInfo ( ) . getInstanceName ( ) ; LOG . error ( "Failed executing stage '{}': {}" , instanceName , t . toString ( ) , t ) ; Throwables . propagateIfInstanceOf ( t , PipelineRuntimeException . class ) ; Throwables . propagateIfInstanceOf ( t , StageException . class ) ; Throwables . propagate ( t ) ; } }


Input: Evaluate and obtain the row id if the expression is present or return null . [CODESPLIT] private String getInsertIdForRecord ( ELVars elVars , Record record ) throws OnRecordErrorException { String recordId = null ; RecordEL . setRecordInContext ( elVars , record ) ; try { if ( ! ( StringUtils . isEmpty ( conf . rowIdExpression ) ) ) { recordId = rowIdELEval . eval ( elVars , conf . rowIdExpression , String . class ) ; if ( StringUtils . isEmpty ( recordId ) ) { throw new OnRecordErrorException ( record , Errors . BIGQUERY_15 ) ; } } } catch ( ELEvalException e ) { LOG . error ( "Error evaluating Row Expression EL" , e ) ; throw new OnRecordErrorException ( record , Errors . BIGQUERY_10 , e ) ; } return recordId ; }


Input: Convert the root field to a java map object implicitly mapping each field to the column ( only non nested objects ) [CODESPLIT] private Map < String , Object > convertToRowObjectFromRecord ( Record record ) throws OnRecordErrorException { Field rootField = record . get ( ) ; Map < String , Object > rowObject = new LinkedHashMap <> ( ) ; if ( rootField . getType ( ) . isOneOf ( Field . Type . MAP , Field . Type . LIST_MAP ) ) { Map < String , Field > fieldMap = rootField . getValueAsMap ( ) ; for ( Map . Entry < String , Field > fieldEntry : fieldMap . entrySet ( ) ) { Field field = fieldEntry . getValue ( ) ; //Skip null value fields if ( field . getValue ( ) != null ) { try { rowObject . put ( fieldEntry . getKey ( ) , getValueFromField ( "/" + fieldEntry . getKey ( ) , field ) ) ; } catch ( IllegalArgumentException e ) { throw new OnRecordErrorException ( record , Errors . BIGQUERY_13 , e . getMessage ( ) ) ; } } } } else { throw new OnRecordErrorException ( record , Errors . BIGQUERY_16 ) ; } return rowObject ; }


Input: Convert the sdc Field to an object for row content [CODESPLIT] private Object getValueFromField ( String fieldPath , Field field ) { LOG . trace ( "Visiting Field Path '{}' of type '{}'" , fieldPath , field . getType ( ) ) ; switch ( field . getType ( ) ) { case LIST : //REPEATED List < Field > listField = field . getValueAsList ( ) ; //Convert the list to map with indices as key and Field as value (Map<Integer, Field>) Map < Integer , Field > fields = IntStream . range ( 0 , listField . size ( ) ) . boxed ( ) . collect ( Collectors . toMap ( Function . identity ( ) , listField :: get ) ) ; //filter map to remove fields with null value fields = fields . entrySet ( ) . stream ( ) . filter ( e -> e . getValue ( ) . getValue ( ) != null ) . collect ( Collectors . toMap ( Map . Entry :: getKey , Map . Entry :: getValue ) ) ; //now use the map index to generate field path and generate object for big query write return fields . entrySet ( ) . stream ( ) . map ( e -> getValueFromField ( fieldPath + "[" + e . getKey ( ) + "]" , e . getValue ( ) ) ) . collect ( Collectors . toList ( ) ) ; case MAP : case LIST_MAP : //RECORD return field . getValueAsMap ( ) . entrySet ( ) . stream ( ) . filter ( me -> me . getValue ( ) . getValue ( ) != null ) . collect ( Collectors . toMap ( Map . Entry :: getKey , e -> getValueFromField ( fieldPath + "/" + e . getKey ( ) , e . getValue ( ) ) ) ) ; case DATE : return dateFormat . format ( field . getValueAsDate ( ) ) ; case TIME : return timeFormat . format ( field . getValueAsTime ( ) ) ; case DATETIME : return dateTimeFormat . format ( field . getValueAsDatetime ( ) ) ; case BYTE_ARRAY : return Base64 . getEncoder ( ) . encodeToString ( field . getValueAsByteArray ( ) ) ; case DECIMAL : case BYTE : case CHAR : case FILE_REF : throw new IllegalArgumentException ( Utils . format ( Errors . BIGQUERY_12 . getMessage ( ) , fieldPath , field . getType ( ) ) ) ; default : //Boolean -> Map to Boolean in big query //Float, Double -> Map to Float in big query //String -> maps to String in big query //Short, Integer, Long -> Map to integer in big query return field . getValue ( ) ; } }


Input: port . [CODESPLIT] private void checkValidPorts ( ) { if ( ( conf . get ( HTTP_PORT_KEY , HTTP_PORT_DEFAULT ) == 0 && conf . get ( HTTPS_PORT_KEY , HTTPS_PORT_DEFAULT ) != - 1 ) || ( conf . get ( HTTPS_PORT_KEY , HTTPS_PORT_DEFAULT ) == 0 && conf . get ( HTTP_PORT_KEY , HTTP_PORT_DEFAULT ) != - 1 ) ) { throw new IllegalArgumentException ( "Invalid port combination for http and https, If http port is set to 0 (random), then https should be " + "set to -1 or vice versa" ) ; } }


Input: Aggregates errors that occur during the processing of a batch . [CODESPLIT] private < T > Observable < T > handleError ( Record record , Errors error , Throwable ex , boolean passable ) { errors . put ( record , new ErrorRecord ( error , ex , passable ) ) ; return Observable . empty ( ) ; }


Input: Aggregates errors that occur during the processing of a batch . [CODESPLIT] private < T > Observable < T > handleError ( Record record , Errors error , boolean passable ) { return handleError ( record , error , new RuntimeException ( ) , passable ) ; }


Input: Writes sub - document lookup values to the record [CODESPLIT] private Observable < Record > setFragmentInRecord ( Record record , DocumentFragment < Lookup > frag ) { if ( frag . content ( 0 ) == null ) { LOG . debug ( "Sub-document path not found" ) ; return handleError ( record , Errors . COUCHBASE_25 , true ) ; } for ( SubdocMappingConfig subdocMapping : config . subdocMappingConfigs ) { Object fragJson = frag . content ( subdocMapping . subdocPath ) ; if ( fragJson == null ) { return handleError ( record , Errors . COUCHBASE_25 , true ) ; } try { record . set ( subdocMapping . sdcField , jsonToField ( fragJson ) ) ; record . getHeader ( ) . setAttribute ( config . CAS_HEADER_ATTRIBUTE , String . valueOf ( frag . cas ( ) ) ) ; } catch ( IOException e ) { try { record . set ( subdocMapping . sdcField , jsonToField ( JsonObject . fromJson ( fragJson . toString ( ) ) . toMap ( ) ) ) ; record . getHeader ( ) . setAttribute ( config . CAS_HEADER_ATTRIBUTE , String . valueOf ( frag . cas ( ) ) ) ; } catch ( IOException ex ) { return handleError ( record , Errors . COUCHBASE_19 , ex , false ) ; } } } return Observable . just ( record ) ; }


Input: Writes full document lookup values to the record [CODESPLIT] private Observable < Record > setDocumentInRecord ( Record record , JsonDocument doc ) { if ( doc . content ( ) == null ) { LOG . debug ( "Document does not exist: {}" , doc . id ( ) ) ; return handleError ( record , Errors . COUCHBASE_26 , true ) ; } try { record . set ( config . outputField , jsonToField ( doc . content ( ) . toMap ( ) ) ) ; record . getHeader ( ) . setAttribute ( config . CAS_HEADER_ATTRIBUTE , String . valueOf ( doc . cas ( ) ) ) ; return Observable . just ( record ) ; } catch ( IOException e ) { LOG . debug ( "Unable to set KV lookup in record for: {}" , doc . id ( ) ) ; return handleError ( record , Errors . COUCHBASE_19 , e , false ) ; } }


Input: Writes N1QL query result rows to the record [CODESPLIT] private Observable < Record > setN1QLRowInRecord ( Record record , AsyncN1qlQueryRow row ) { for ( N1QLMappingConfig n1qlMapping : config . n1qlMappingConfigs ) { if ( config . multipleValueOperation == MultipleValueType . FIRST && record . get ( n1qlMapping . sdcField ) != null ) { LOG . debug ( "Only populating output field with first record. Skipping additional result." ) ; return Observable . empty ( ) ; } Object property = row . value ( ) . get ( n1qlMapping . property ) ; if ( property == null ) { LOG . debug ( "Requested property not returned: {}" , n1qlMapping . property ) ; return handleError ( record , Errors . COUCHBASE_27 , true ) ; } try { record . set ( n1qlMapping . sdcField , jsonToField ( property ) ) ; } catch ( IOException e ) { try { record . set ( n1qlMapping . sdcField , jsonToField ( JsonObject . fromJson ( property . toString ( ) ) . toMap ( ) ) ) ; } catch ( IOException ex ) { LOG . debug ( "Unable to set N1QL property in record" ) ; return handleError ( record , Errors . COUCHBASE_19 , ex , false ) ; } } } return Observable . just ( record ) ; }


Input: Take an numeric operation code and check if the number is valid operation code . The operation code must be numeric : 1 ( insert ) 2 ( update ) 3 ( delete ) etc [CODESPLIT] public static int convertToJDBCCode ( int op ) { if ( CRUD_MAP . containsKey ( op ) ) { return CRUD_MAP . get ( op ) ; } throw new UnsupportedOperationException ( Utils . format ( "Operation code {} is not supported" , op ) ) ; }


Input: as the record is just the metadata along with file ref . [CODESPLIT] private void handleWholeFileDataFormat ( S3ObjectSummary s3ObjectSummary , String recordId ) throws StageException { S3Object partialS3ObjectForMetadata ; //partialObject with fetchSize 1 byte. //This is mostly used for extracting metadata and such. partialS3ObjectForMetadata = AmazonS3Util . getObjectRange ( s3Client , s3ConfigBean . s3Config . bucket , s3ObjectSummary . getKey ( ) , 1 , s3ConfigBean . sseConfig . useCustomerSSEKey , s3ConfigBean . sseConfig . customerKey , s3ConfigBean . sseConfig . customerKeyMd5 ) ; S3FileRef . Builder s3FileRefBuilder = new S3FileRef . Builder ( ) . s3Client ( s3Client ) . s3ObjectSummary ( s3ObjectSummary ) . useSSE ( s3ConfigBean . sseConfig . useCustomerSSEKey ) . customerKey ( s3ConfigBean . sseConfig . customerKey ) . customerKeyMd5 ( s3ConfigBean . sseConfig . customerKeyMd5 ) . bufferSize ( ( int ) dataParser . suggestedWholeFileBufferSize ( ) ) . createMetrics ( true ) . totalSizeInBytes ( s3ObjectSummary . getSize ( ) ) . rateLimit ( dataParser . wholeFileRateLimit ( ) ) ; if ( dataParser . isWholeFileChecksumRequired ( ) ) { s3FileRefBuilder . verifyChecksum ( true ) . checksumAlgorithm ( HashingUtil . HashType . MD5 ) //128 bit hex encoded md5 checksum. . checksum ( partialS3ObjectForMetadata . getObjectMetadata ( ) . getETag ( ) ) ; } Map < String , Object > metadata = AmazonS3Util . getMetaData ( partialS3ObjectForMetadata ) ; metadata . put ( S3Constants . BUCKET , s3ObjectSummary . getBucketName ( ) ) ; metadata . put ( S3Constants . OBJECT_KEY , s3ObjectSummary . getKey ( ) ) ; metadata . put ( S3Constants . OWNER , s3ObjectSummary . getOwner ( ) ) ; metadata . put ( S3Constants . SIZE , s3ObjectSummary . getSize ( ) ) ; metadata . put ( HeaderAttributeConstants . FILE_NAME , s3ObjectSummary . getKey ( ) ) ; metadata . remove ( S3Constants . CONTENT_LENGTH ) ; parser = dataParser . getParser ( recordId , metadata , s3FileRefBuilder . build ( ) ) ; //Object is assigned so that setHeaders() function can use this to get metadata //information about the object object = partialS3ObjectForMetadata ; }


Input: Initialize the gauge with needed information [CODESPLIT] private void initGaugeIfNeeded ( ) { gaugeMap . put ( S3Constants . THREAD_NAME , Thread . currentThread ( ) . getName ( ) ) ; gaugeMap . put ( S3Constants . STATUS , "" ) ; gaugeMap . put ( S3Constants . BUCKET , "" ) ; gaugeMap . put ( S3Constants . OBJECT_KEY , "" ) ; }


Input: Check if given gtid + seqNo pair is contained in this incomplete transactions . [CODESPLIT] public boolean incompleteTransactionsContain ( String gtid , long seqNo ) { Long s = incompleteTransactions . get ( gtid ) ; return s != null && s >= seqNo ; }


Input: Runs a Supplier within the context of a specified ClassLoader . [CODESPLIT] public static < T > T withClassLoader ( ClassLoader classLoader , ExceptionSupplier < T > supplier ) { try { return withClassLoaderInternal ( classLoader , supplier ) ; } catch ( Exception e ) { Throwables . propagate ( e ) ; } return null ; }


Input: Runs a Supplier within the context of a specified ClassLoader . [CODESPLIT] public static < T , E1 extends Exception > T withClassLoader ( ClassLoader classLoader , Class < E1 > e1 , ExceptionSupplier < T > supplier ) throws E1 { try { return withClassLoaderInternal ( classLoader , supplier ) ; } catch ( Exception e ) { Throwables . propagateIfPossible ( e , e1 ) ; Throwables . propagate ( e ) ; } return null ; }


Input: Runs a Supplier within the context of a specified ClassLoader and in priviledged mode . [CODESPLIT] public static < T > T privilegedWithClassLoader ( ClassLoader classLoader , ExceptionSupplier < T > supplier ) { try { return AccessController . doPrivileged ( ( PrivilegedExceptionAction < T > ) ( ) -> withClassLoaderInternal ( classLoader , supplier ) ) ; } catch ( PrivilegedActionException e ) { Throwables . propagate ( e ) ; } return null ; }


Input: Runs a Supplier within the context of a specified ClassLoader and in priviledged mode . [CODESPLIT] public static < T , E1 extends Exception > T privilegedWithClassLoader ( ClassLoader classLoader , Class < E1 > e1 , ExceptionSupplier < T > supplier ) throws E1 { try { return AccessController . doPrivileged ( ( PrivilegedExceptionAction < T > ) ( ) -> withClassLoaderInternal ( classLoader , supplier ) ) ; } catch ( PrivilegedActionException e ) { Throwables . propagateIfPossible ( e . getCause ( ) , e1 ) ; Throwables . propagate ( e ) ; } return null ; }


Input: Internal version of the wrapping function that will simply propagate all exceptions up . [CODESPLIT] private static < T > T withClassLoaderInternal ( ClassLoader classLoader , ExceptionSupplier < T > supplier ) throws Exception { ClassLoader previousClassLoader = Thread . currentThread ( ) . getContextClassLoader ( ) ; try { Thread . currentThread ( ) . setContextClassLoader ( classLoader ) ; return supplier . get ( ) ; } finally { Thread . currentThread ( ) . setContextClassLoader ( previousClassLoader ) ; } }


Input: HeaderImpl setter methods [CODESPLIT] public void setStageCreator ( String stateCreator ) { Preconditions . checkNotNull ( stateCreator , "stateCreator cannot be null" ) ; map . put ( STAGE_CREATOR_INSTANCE_ATTR , stateCreator ) ; }


Input: To be removed [CODESPLIT] public Map < String , Object > getUserAttributes ( ) { return map . entrySet ( ) . stream ( ) . filter ( map -> ! map . getKey ( ) . startsWith ( RESERVED_PREFIX ) ) . collect ( Collectors . toMap ( map -> map . getKey ( ) , map -> map . getValue ( ) ) ) ; }


Input: To be removed [CODESPLIT] public Map < String , Object > setUserAttributes ( Map < String , Object > newAttributes ) { // ImmutableMap can't have null values and our map could have, so use unmodifiable map Map < String , Object > old = Collections . unmodifiableMap ( getUserAttributes ( ) ) ; //Set current map to just the Reserved System Attributes map = getSystemAttributes ( ) ; // Add and validate each of the new user attributes newAttributes . forEach ( ( k , v ) -> setAttribute ( k , v . toString ( ) ) ) ; return old ; }


Input: Logs in . If Kerberos is enabled it logs in against the KDC otherwise is a NOP . [CODESPLIT] public synchronized void login ( ) { if ( subject != null ) { throw new IllegalStateException ( Utils . format ( "Service already login, Principal '{}'" , subject . getPrincipals ( ) ) ) ; } if ( securityConfiguration . isKerberosEnabled ( ) ) { try { loginContext = createLoginContext ( ) ; subject = loginContext . getSubject ( ) ; } catch ( Exception ex ) { throw new RuntimeException ( Utils . format ( "Could not get Kerberos credentials: {}" , ex . toString ( ) ) , ex ) ; } if ( renewalThread == null ) { renewalThread = new Thread ( ) { @ Override public void run ( ) { LOG . debug ( "Starting renewal thread" ) ; if ( ! SecurityContext . this . sleep ( THIRTY_SECONDS_MS ) ) { LOG . info ( "Interrupted, exiting renewal thread" ) ; return ; } while ( true ) { LOG . trace ( "Renewal check starts" ) ; try { KerberosTicket lastExpiringTGT = getNewestTGT ( ) ; if ( lastExpiringTGT == null ) { LOG . warn ( "Could not obtain kerberos ticket, it may have expired already or it was logged out, will wait" + "30 secs to attempt a relogin" ) ; LOG . trace ( "Ticket not found, sleeping 30 secs and trying to login" ) ; if ( ! SecurityContext . this . sleep ( THIRTY_SECONDS_MS ) ) { LOG . info ( "Interrupted, exiting renewal thread" ) ; return ; } } else { long renewalTimeMs = calculateRenewalTime ( lastExpiringTGT ) - THIRTY_SECONDS_MS ; LOG . trace ( "Ticket found time to renewal '{}ms', sleeping that time" , renewalTimeMs ) ; if ( renewalTimeMs > 0 ) { if ( ! SecurityContext . this . sleep ( renewalTimeMs ) ) { LOG . info ( "Interrupted, exiting renewal thread" ) ; return ; } } } LOG . debug ( "Triggering relogin" ) ; Set < KerberosTicket > oldTickets = getSubject ( ) . getPrivateCredentials ( KerberosTicket . class ) ; relogin ( 3 ) ; // Remove all old private credentials, since we only need the new one we just added getSubject ( ) . getPrivateCredentials ( ) . removeAll ( oldTickets ) ; } catch ( Exception exception ) { LOG . error ( "Stopping renewal thread because of exception: " + exception , exception ) ; return ; } catch ( Throwable throwable ) { LOG . error ( "Error in renewal thread: " + throwable , throwable ) ; return ; } } } } ; List < String > principals = new ArrayList <> ( ) ; for ( Principal p : subject . getPrincipals ( ) ) { principals . add ( p . getName ( ) ) ; } renewalThread . setName ( "Kerberos-Renewal-Thread-" + Joiner . on ( "," ) . join ( principals ) ) ; renewalThread . setContextClassLoader ( Thread . currentThread ( ) . getContextClassLoader ( ) ) ; renewalThread . setDaemon ( true ) ; renewalThread . start ( ) ; } } else { subject = new Subject ( ) ; } LOG . debug ( "Login. Kerberos enabled '{}', Principal '{}'" , securityConfiguration . isKerberosEnabled ( ) , subject . getPrincipals ( ) ) ; }


Input: Logs out . If Keberos is enabled it logs out from the KDC otherwise is a NOP . [CODESPLIT] public synchronized void logout ( ) { if ( subject != null ) { LOG . debug ( "Logout. Kerberos enabled '{}', Principal '{}'" , securityConfiguration . isKerberosEnabled ( ) , subject . getPrincipals ( ) ) ; if ( loginContext != null ) { try { loginContext . logout ( ) ; } catch ( LoginException ex ) { LOG . warn ( "Error while doing logout from Kerberos: {}" , ex . toString ( ) , ex ) ; } finally { loginContext = null ; } } subject = null ; } }


Input: This method should be called only once and before any stages are loaded . [CODESPLIT] private void setExceptions ( Configuration configuration ) { this . exceptions . clear ( ) ; this . stageLibExceptions . clear ( ) ; // Load general exceptions for ( String path : configuration . get ( PROPERTY_EXCEPTIONS , "" ) . split ( "," ) ) { this . exceptions . add ( replaceVariables ( path ) ) ; } // Load Stage library specific exceptions Configuration stageSpecific = configuration . getSubSetConfiguration ( PROPERTY_STAGE_EXCEPTIONS , true ) ; for ( Map . Entry < String , String > entry : stageSpecific . getValues ( ) . entrySet ( ) ) { Set < String > stageExceptions = new HashSet <> ( ) ; for ( String path : entry . getValue ( ) . split ( "," ) ) { stageExceptions . add ( replaceVariables ( path ) ) ; } this . stageLibExceptions . put ( entry . getKey ( ) , stageExceptions ) ; } }


Input: Replace variables to internal SDC directories so that users don t have to be entering FQDN . [CODESPLIT] private String replaceVariables ( String path ) { return path . replace ( "$SDC_DATA" , dataDir ) . replace ( "$SDC_CONF" , configDir ) . replace ( "$SDC_RESOURCES" , resourcesDir ) ; }


Input: Make sure that the active code have proper rights to access the file inside protected directory . [CODESPLIT] private void ensureProperPermissions ( String path ) { ClassLoader cl = Thread . currentThread ( ) . getContextClassLoader ( ) ; // 1) Container can access anything if ( cl instanceof ContainerClassLoader ) { return ; } // 2. Some files are whitelisted globally for all stage libraries if ( exceptions . contains ( path ) ) { return ; } // 3. Some stage libraries have some files whitelisted globally if ( cl instanceof SDCClassLoader ) { String libraryName = ( ( SDCClassLoader ) cl ) . getName ( ) ; if ( stageLibExceptions . containsKey ( libraryName ) && stageLibExceptions . get ( libraryName ) . contains ( path ) ) { return ; } } // No whitelist, no fun, go away throw new SecurityException ( Utils . format ( "Classloader {} is not allowed access to Data Collector internal directories ({})." , cl . toString ( ) , path ) ) ; }


Input: Generate Hive Type Info Representation inside the Metadata Record . [CODESPLIT] public Field generateJdbcTypeInfoFieldForMetadataRecord ( JdbcTypeInfo jdbcTypeInfo ) { Map < String , Field > fields = new HashMap <> ( ) ; fields . put ( TYPE , Field . create ( jdbcTypeInfo . getJdbcType ( ) . name ( ) ) ) ; fields . put ( EXTRA_INFO , generateExtraInfoFieldForMetadataRecord ( jdbcTypeInfo ) ) ; return Field . create ( fields ) ; }


Input: Generate { @link JdbcTypeInfo } from the Metadata Record <br > . ( Reverse of { @link #generateJdbcTypeInfoFieldForMetadataRecord ( JdbcTypeInfo ) } ) [CODESPLIT] @ SuppressWarnings ( "unchecked" ) public JdbcTypeInfo generateJdbcTypeInfoFromMetadataField ( Field jdbcTypeInfoField , JdbcSchemaWriter schemaWriter ) throws StageException { if ( jdbcTypeInfoField . getType ( ) == Field . Type . MAP ) { Map < String , Field > fields = ( Map < String , Field > ) jdbcTypeInfoField . getValue ( ) ; if ( ! fields . containsKey ( TYPE ) || ! fields . containsKey ( EXTRA_INFO ) ) { throw new StageException ( JdbcErrors . JDBC_308 , TYPE_INFO ) ; } JdbcType jdbcType = JdbcType . getJdbcTypeFromString ( fields . get ( TYPE ) . getValueAsString ( ) ) ; return generateJdbcTypeInfoFromMetadataField ( jdbcType , fields . get ( EXTRA_INFO ) , schemaWriter ) ; } else { throw new StageException ( JdbcErrors . JDBC_308 , TYPE_INFO ) ; } }


Input: Generate Column Definition for create table / add columns [CODESPLIT] public String generateColumnTypeDefinition ( JdbcTypeInfo jdbcTypeInfo , String columnName ) { return String . format ( COLUMN_TYPE , columnName , jdbcTypeInfo . toString ( ) ) ; }


Input: Obtaining a reference on the dummy source which is used to feed a pipeline<br / > Direction : Stage - > Container [CODESPLIT] public static /*PipelineStartResult*/ Object startPipeline ( Runnable postBatchRunnable ) throws Exception { BootstrapCluster . initialize ( ) ; ClassLoader originalClassLoader = Thread . currentThread ( ) . getContextClassLoader ( ) ; try { Thread . currentThread ( ) . setContextClassLoader ( containerCL ) ; Class embeddedPipelineFactoryClz = Class . forName ( "com.streamsets.datacollector.EmbeddedDataCollectorFactory" , true , containerCL ) ; Method createPipelineMethod = embeddedPipelineFactoryClz . getMethod ( "startPipeline" , Runnable . class ) ; return createPipelineMethod . invoke ( null , postBatchRunnable ) ; } catch ( Exception ex ) { String msg = "Error trying to create pipeline: " + ex ; throw new IllegalStateException ( msg , ex ) ; } finally { Thread . currentThread ( ) . setContextClassLoader ( originalClassLoader ) ; } }


Input: Bootstrapping the Driver which starts a EMR job on cluster [CODESPLIT] public static void main ( String [ ] args ) throws Exception { EmrBinding binding = null ; try { binding = new EmrBinding ( args ) ; binding . init ( ) ; binding . awaitTermination ( ) ; // killed by ClusterProviderImpl before returning } catch ( Exception ex ) { String msg = "Error trying to invoke BootstrapEmrBatch.main: " + ex ; throw new IllegalStateException ( msg , ex ) ; } finally { try { if ( binding != null ) { binding . close ( ) ; } } catch ( Exception ex ) { LOG . warn ( "Error on binding close: " + ex , ex ) ; } } }


Input: Returns directory path for given record and date . [CODESPLIT] String getDirPath ( Date date , Record record ) throws StageException { if ( dirPathTemplateInHeader ) { // We're not validating if the header exists as that job is already done return record . getHeader ( ) . getAttribute ( HdfsTarget . TARGET_DIRECTORY_HEADER ) ; } return pathResolver . resolvePath ( date , record ) ; }


Input: This method should be called every time we finish writing into a file and consider it done . [CODESPLIT] Path renameToFinalName ( FileSystem fs , Path tempPath ) throws IOException , StageException { return fsHelper . renameAndGetPath ( fs , tempPath ) ; }


Input: Produce events that were cached during the batch processing . [CODESPLIT] public void issueCachedEvents ( ) throws IOException { Path closedPath ; while ( ( closedPath = closedPaths . poll ( ) ) != null ) { produceCloseFileEvent ( fs , closedPath ) ; } }


Input: rename all _tmp_ files under directory path ( dirPathTemplate ) return the number of _tmp_ files [CODESPLIT] public int handleAlreadyExistingFiles ( ) throws StageException , IOException { int result = 0 ; String globPath = dirPathTemplate ; final String staticExpReg = "\\$\\{(sdc:|pipeline:|runtime:)[a-zA-Z0-9\\(\\)]*\\}" ; Pattern pattern = Pattern . compile ( staticExpReg ) ; Matcher matcher = pattern . matcher ( globPath ) ; ELEval eval = context . createELEval ( "dirPathTemplate" ) ; ELVars vars = context . createELVars ( ) ; while ( matcher . find ( ) ) { String expressionString = eval . eval ( vars , matcher . group ( ) , String . class ) ; globPath = globPath . replace ( matcher . group ( ) , expressionString ) ; } final String expReg = "\\$\\{[^}]*\\}" ; pattern = Pattern . compile ( expReg ) ; matcher = pattern . matcher ( globPath ) ; while ( matcher . find ( ) ) { globPath = globPath . replace ( matcher . group ( ) , "*" ) ; } globPath = globPath . replaceAll ( "\\*+" , "*" ) ; globPath = globPath + "/" + TMP_FILE_PREFIX + uniquePrefix + "*" ; LOG . info ( "Created the following glob path for file recovery: {}" , globPath ) ; FileStatus [ ] fileStatuses = fs . globStatus ( new Path ( globPath ) ) ; for ( FileStatus fileStatus : fileStatuses ) { fsHelper . handleAlreadyExistingFile ( fs , fileStatus . getPath ( ) ) ; result ++ ; } return result ; }


Input: This method must always be called after the closeLock () method on the writer has been called . [CODESPLIT] public Path commitWriter ( RecordWriter writer ) throws IOException , StageException { Path path = null ; if ( ( ! writer . isClosed ( ) || writer . isIdleClosed ( ) ) && ! writer . isRenamed ( ) ) { // Unset the interrupt flag before close(). InterruptedIOException makes close() fail // resulting that the tmp file never gets renamed when stopping the pipeline. boolean interrupted = Thread . interrupted ( ) ; try { // Since this method is always called from exactly one thread, and // we checked to make sure that it was not closed or it was idle closed, this method either closes // the file or pushes us into the catch block. writer . close ( ) ; } catch ( IdleClosedException e ) { LOG . info ( "Writer for {} was idle closed, renaming.." , writer . getPath ( ) ) ; } LOG . debug ( "Path[{}] - Committing Writer" , writer . getPath ( ) ) ; path = renameToFinalName ( fs , writer . getPath ( ) ) ; writer . setRenamed ( true ) ; LOG . debug ( "Path[{}] - Committed Writer to '{}'" , writer . getPath ( ) , path ) ; // Reset the interrupt flag back. if ( interrupted ) { Thread . currentThread ( ) . interrupt ( ) ; } } return path ; }


Input: Return true if this record should be written into a new file regardless whether we have a file for the record currently opened or not . [CODESPLIT] public boolean shouldRoll ( RecordWriter writer , Record record ) { if ( rollIfHeader && record . getHeader ( ) . getAttribute ( rollHeaderName ) != null ) { LOG . debug ( "Path[{}] - will be rolled because of roll attribute '{}' set to '{}' in the record : '{}'" , writer . getPath ( ) , rollHeaderName , record . getHeader ( ) . getAttribute ( rollHeaderName ) , record . getHeader ( ) . getSourceId ( ) ) ; return true ; } return false ; }


Input: / * When we start with a file ( empty or not ) the file offset is zero . If the file is a rolled file the file will be EOF immediately triggering a close of the reader and setting the offset to Long . MAX_VALUE ( this happens in the MultiDirectoryReader class ) . This is the signal that in the next read a directory scan should be triggered to get the next rolled file or the live file if we were scanning the last rolled file . If the file you are starting is the live file we don t get an EOF as we expect data to be appended . We just return null chunks while there is no data . If the file is rolled we ll detect that and then do what is described in the previous paragraph . [CODESPLIT] @ Override public String produce ( String lastSourceOffset , int maxBatchSize , BatchMaker batchMaker ) throws StageException { int recordCounter = 0 ; long startTime = System . currentTimeMillis ( ) ; maxBatchSize = Math . min ( conf . batchSize , maxBatchSize ) ; // deserializing offsets of all directories Map < String , String > offsetMap = deserializeOffsetMap ( lastSourceOffset ) ; boolean offsetSet = false ; while ( ! offsetSet ) { try { multiDirReader . setOffsets ( offsetMap ) ; offsetSet = true ; } catch ( IOException ex ) { LOG . warn ( "Error while creating reading previous offset: {}" , ex . toString ( ) , ex ) ; multiDirReader . purge ( ) ; } } while ( recordCounter < maxBatchSize && ! isTimeout ( startTime ) ) { LiveFileChunk chunk = multiDirReader . next ( getRemainingWaitTime ( startTime ) ) ; if ( chunk != null ) { String tag = chunk . getTag ( ) ; tag = ( tag != null && tag . isEmpty ( ) ) ? null : tag ; String liveFileStr = chunk . getFile ( ) . serialize ( ) ; List < FileLine > lines = chunk . getLines ( ) ; int truncatedLine = chunk . isTruncated ( ) ? lines . size ( ) - 1 : - 1 ; for ( int i = 0 ; i < lines . size ( ) ; i ++ ) { FileLine line = lines . get ( i ) ; String sourceId = liveFileStr + "::" + line . getFileOffset ( ) ; try ( DataParser parser = parserFactory . getParser ( sourceId , line . getText ( ) ) ) { if ( i == truncatedLine ) { //set truncated parser . setTruncated ( ) ; } Record record = parser . parse ( ) ; if ( record != null ) { if ( tag != null ) { record . getHeader ( ) . setAttribute ( "tag" , tag ) ; } record . getHeader ( ) . setAttribute ( HeaderAttributeConstants . FILE , chunk . getFile ( ) . getPath ( ) . toString ( ) ) ; record . getHeader ( ) . setAttribute ( HeaderAttributeConstants . FILE_NAME , chunk . getFile ( ) . getPath ( ) . getFileName ( ) . toString ( ) ) ; record . getHeader ( ) . setAttribute ( HeaderAttributeConstants . OFFSET , String . valueOf ( line . getFileOffset ( ) ) ) ; record . getHeader ( ) . setAttribute ( HeaderAttributeConstants . LAST_MODIFIED_TIME , String . valueOf ( Files . getLastModifiedTime ( chunk . getFile ( ) . getPath ( ) ) . toMillis ( ) ) ) ; batchMaker . addRecord ( record , outputLane ) ; recordCounter ++ ; } } catch ( IOException | DataParserException ex ) { errorRecordHandler . onError ( Errors . TAIL_12 , sourceId , ex . toString ( ) , ex ) ; } } } } boolean metadataGenerationFailure = false ; Date now = new Date ( startTime ) ; for ( FileEvent event : multiDirReader . getEvents ( ) ) { try { LiveFile file = event . getFile ( ) . refresh ( ) ; Record metadataRecord = getContext ( ) . createRecord ( "" ) ; Map < String , Field > map = new HashMap <> ( ) ; map . put ( "fileName" , Field . create ( file . getPath ( ) . toString ( ) ) ) ; map . put ( "inode" , Field . create ( file . getINode ( ) ) ) ; map . put ( "time" , Field . createDate ( now ) ) ; map . put ( "event" , Field . create ( ( event . getAction ( ) . name ( ) ) ) ) ; metadataRecord . set ( Field . create ( map ) ) ; batchMaker . addRecord ( metadataRecord , metadataLane ) ; // We're also sending the same information on event lane String eventRecordSourceId = Utils . format ( "event:{}:{}:{}" , event . getAction ( ) . name ( ) , 1 , file . getPath ( ) . toString ( ) ) ; EventRecord eventRecord = getContext ( ) . createEventRecord ( event . getAction ( ) . name ( ) , 1 , eventRecordSourceId ) ; eventRecord . set ( Field . create ( map ) ) ; getContext ( ) . toEvent ( eventRecord ) ; } catch ( IOException ex ) { LOG . warn ( "Error while creating metadata records: {}" , ex . toString ( ) , ex ) ; metadataGenerationFailure = true ; } } if ( metadataGenerationFailure ) { multiDirReader . purge ( ) ; } boolean offsetExtracted = false ; while ( ! offsetExtracted ) { try { offsetMap = multiDirReader . getOffsets ( ) ; offsetExtracted = true ; } catch ( IOException ex ) { LOG . warn ( "Error while creating creating new offset: {}" , ex . toString ( ) , ex ) ; multiDirReader . purge ( ) ; } } //Calculate Offset lag Metric. calculateOffsetLagMetric ( offsetMap ) ; //Calculate Pending Files Metric calculatePendingFilesMetric ( ) ; // serializing offsets of all directories return serializeOffsetMap ( offsetMap ) ; }


Input: Set the path separator to use for pattern parsing . <p > Default is / as in Ant . [CODESPLIT] public void setPathSeparator ( String pathSeparator ) { this . pathSeparator = ( pathSeparator != null ? pathSeparator : DEFAULT_PATH_SEPARATOR ) ; this . pathSeparatorPatternCache = new PathSeparatorPatternCache ( this . pathSeparator ) ; }


Input: Tokenize the given path pattern into parts based on this matcher s settings . <p > Performs caching based on { [CODESPLIT] protected String [ ] tokenizePattern ( String pattern ) { String [ ] tokenized = null ; Boolean cachePatterns = this . cachePatterns ; if ( cachePatterns == null || cachePatterns . booleanValue ( ) ) { tokenized = this . tokenizedPatternCache . get ( pattern ) ; } if ( tokenized == null ) { tokenized = tokenizePath ( pattern ) ; if ( cachePatterns == null && this . tokenizedPatternCache . size ( ) >= CACHE_TURNOFF_THRESHOLD ) { // Try to adapt to the runtime situation that we're encountering: // There are obviously too many different patterns coming in here... // So let's turn off the cache since the patterns are unlikely to be reoccurring. deactivatePatternCache ( ) ; return tokenized ; } if ( cachePatterns == null || cachePatterns . booleanValue ( ) ) { this . tokenizedPatternCache . put ( pattern , tokenized ) ; } } return tokenized ; }


Input: Copy the given { [CODESPLIT] private static String [ ] toStringArray ( Collection < String > collection ) { if ( collection == null ) { return null ; } return collection . toArray ( new String [ collection . size ( ) ] ) ; }


Input: Test whether or not a string matches against a pattern . [CODESPLIT] private boolean matchStrings ( String pattern , String str , Map < String , String > uriTemplateVariables ) { return getStringMatcher ( pattern ) . matchStrings ( str , uriTemplateVariables ) ; }


Input: Sets the file offsets to use for the next read . To work correctly the last return offsets should be used or an empty <code > Map< / code > if there is none . <p / > If a reader is already live the corresponding set offset is ignored as we cache all the contextual information of live readers . [CODESPLIT] public void setOffsets ( Map < String , String > offsets ) throws IOException { Utils . checkState ( open , "Not open" ) ; fileContextProvider . setOffsets ( offsets ) ; // we reset the events on every setOffsets(). events . clear ( ) ; }


Input: Returns the current file offsets . The returned offsets should be set before the next read . [CODESPLIT] public Map < String , String > getOffsets ( ) throws IOException { Utils . checkState ( open , "Not open" ) ; return fileContextProvider . getOffsets ( ) ; }


Input: remaining time till timeout return zero if already in timeout [CODESPLIT] private long getRemainingWaitTime ( long startTime , long maxWaitTimeMillis ) { long remaining = maxWaitTimeMillis - ( System . currentTimeMillis ( ) - startTime ) ; return ( remaining > 0 ) ? remaining : 0 ; }


Input: Reads the next { @link LiveFileChunk } from the directories waiting the specified time for one . [CODESPLIT] public LiveFileChunk next ( long waitMillis ) { Utils . checkState ( open , "Not open" ) ; waitMillis = ( waitMillis > 0 ) ? waitMillis : 0 ; long startTime = System . currentTimeMillis ( ) ; LiveFileChunk chunk = null ; boolean exit = false ; fileContextProvider . startNewLoop ( ) ; while ( ! exit ) { if ( ! fileContextProvider . didFullLoop ( ) ) { FileContext fileContext = fileContextProvider . next ( ) ; try { LiveFileReader reader = fileContext . getReader ( ) ; if ( reader != null ) { if ( reader . hasNext ( ) ) { chunk = reader . next ( 0 ) ; if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "next(): directory '{}', file '{}', offset '{}' got data '{}'" , fileContext . getMultiFileInfo ( ) . getFileFullPath ( ) , reader . getLiveFile ( ) , reader . getOffset ( ) , chunk != null ) ; } } else { if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "next(): directory '{}', file '{}', offset '{}' EOF reached" , fileContext . getMultiFileInfo ( ) . getFileFullPath ( ) , reader . getLiveFile ( ) , reader . getOffset ( ) ) ; } } fileContext . releaseReader ( false ) ; } else { if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "next(): directory '{}', no reader available" , fileContext . getMultiFileInfo ( ) . getFileFullPath ( ) ) ; } } } catch ( IOException ex ) { LOG . error ( "Error while reading file: {}" , ex . toString ( ) , ex ) ; try { fileContext . releaseReader ( true ) ; } catch ( IOException ex1 ) { LOG . warn ( "Error while releasing reader in error: {}" , ex1 . toString ( ) , ex1 ) ; } } } // check exit conditions (we have a chunk, or we timed-out waitMillis) exit = chunk != null ; if ( ! exit ) { // if we looped thru all dir contexts in this call we yield CPU if ( fileContextProvider . didFullLoop ( ) ) { exit = isTimeout ( startTime , waitMillis ) ; if ( ! exit && LOG . isTraceEnabled ( ) ) { LOG . trace ( "next(): looped through all directories, yielding CPU" ) ; } exit = exit || ! ThreadUtil . sleep ( Math . min ( getRemainingWaitTime ( startTime , waitMillis ) , MAX_YIELD_TIME ) ) ; fileContextProvider . startNewLoop ( ) ; } } } return chunk ; }


Input: Determines the offset lag for each active file being read . [CODESPLIT] public Map < String , Long > getOffsetsLag ( Map < String , String > offsetMap ) throws IOException { return fileContextProvider . getOffsetsLag ( offsetMap ) ; }


Input: Create and initialize new delegate . [CODESPLIT] public < R > R createAndInitialize ( StageLibraryTask stageLib , Configuration configuration , String stageLibraryName , Class < R > exportedInterface ) { StageLibraryDelegate instance = create ( stageLib , stageLibraryName , exportedInterface ) ; if ( instance == null ) { return null ; } // Create & set context StageLibraryDelegateContext context = new StageLibraryDelegateContext ( configuration ) ; instance . setContext ( context ) ; return ( R ) new StageLibraryDelegateRuntime ( instance . getClass ( ) . getClassLoader ( ) , instance ) ; }


Input: Create new instance of the delegator from given stage library . [CODESPLIT] public StageLibraryDelegate create ( StageLibraryTask stageLib , String stageLibraryName , Class exportedInterface ) { StageLibraryDelegateDefinitition def = stageLib . getStageLibraryDelegateDefinition ( stageLibraryName , exportedInterface ) ; if ( def == null ) { return null ; } return createInstance ( def ) ; }


Input: Create actual instance of delegator . [CODESPLIT] private StageLibraryDelegate createInstance ( StageLibraryDelegateDefinitition def ) { StageLibraryDelegate instance = null ; ClassLoader classLoader = Thread . currentThread ( ) . getContextClassLoader ( ) ; try { Thread . currentThread ( ) . setContextClassLoader ( def . getClassLoader ( ) ) ; instance = def . getKlass ( ) . newInstance ( ) ; } catch ( InstantiationException | IllegalAccessException ex ) { LOG . error ( "Can't create instance of delegator: " + ex . toString ( ) , ex ) ; } finally { Thread . currentThread ( ) . setContextClassLoader ( classLoader ) ; } return instance ; }


Input: private static final Configuration jaasConfig = new org . apache . solr . client . solrj . impl . Krb5HttpClientConfigurer . SolrJaasConfiguration () ; [CODESPLIT] public void configure ( DefaultHttpClient httpClient , SolrParams config ) { super . configure ( httpClient , config ) ; // Begin change for SDC-2962 // Instead of checking existence of JAAS file, do the following if solr kerberos is enabled //if (System.getProperty(LOGIN_CONFIG_PROP) != null) { //String configValue = System.getProperty(LOGIN_CONFIG_PROP); //if (configValue != null) { // logger.info("Setting up SPNego auth with config: " + configValue); final String useSubjectCredsProp = "javax.security.auth.useSubjectCredsOnly" ; String useSubjectCredsVal = System . getProperty ( useSubjectCredsProp ) ; // "javax.security.auth.useSubjectCredsOnly" should be false so that the underlying // authentication mechanism can load the credentials from the JAAS configuration. if ( useSubjectCredsVal == null ) { System . setProperty ( useSubjectCredsProp , "false" ) ; } else if ( ! useSubjectCredsVal . toLowerCase ( Locale . ROOT ) . equals ( "false" ) ) { // Don't overwrite the prop value if it's already been written to something else, // but log because it is likely the Credentials won't be loaded correctly. logger . warn ( "System Property: " + useSubjectCredsProp + " set to: " + useSubjectCredsVal + " not false.  SPNego authentication may not be successful." ) ; } // Change for SDC-2962 //javax.security.auth.login.Configuration.setConfiguration(jaasConfig); //Enable only SPNEGO authentication scheme. AuthSchemeRegistry registry = new AuthSchemeRegistry ( ) ; registry . register ( AuthSchemes . SPNEGO , new SPNegoSchemeFactory ( true , false ) ) ; httpClient . setAuthSchemes ( registry ) ; // Get the credentials from the JAAS configuration rather than here Credentials useJaasCreds = new Credentials ( ) { public String getPassword ( ) { return null ; } public Principal getUserPrincipal ( ) { return null ; } } ; SolrPortAwareCookieSpecFactory cookieFactory = new SolrPortAwareCookieSpecFactory ( ) ; httpClient . getCookieSpecs ( ) . register ( cookieFactory . POLICY_NAME , cookieFactory ) ; httpClient . getParams ( ) . setParameter ( ClientPNames . COOKIE_POLICY , cookieFactory . POLICY_NAME ) ; httpClient . getCredentialsProvider ( ) . setCredentials ( AuthScope . ANY , useJaasCreds ) ; httpClient . addRequestInterceptor ( bufferedEntityInterceptor ) ; //} else { //httpClient.getCredentialsProvider().clear(); //} // } }


Input: <p > Scheduler is not used since a dedicated thread will be running to update the token . Even if the thycotic server is not being used the thread will hit for every refresh interval to get a new access token . So instead <tt > volatile< / tt > variables are used to store the token . So only when the token expires thycotic server will be called to fetch the new token . < / p > <p > There are 2 expire checks to make sure that not more than one thread updates the token in case of expiration [CODESPLIT] public String getAccessToken ( ) { if ( System . currentTimeMillis ( ) > getExpireTime ( ) ) { try { synchronized ( this ) { if ( System . currentTimeMillis ( ) > getExpireTime ( ) ) { accessToken = fetchAccessToken ( ) ; if ( accessToken != null && ! accessToken . isEmpty ( ) ) { expires = ( System . currentTimeMillis ( ) + expires ) ; } } } } catch ( Exception e ) { LOG . debug ( "Error in fetching the access token: {} " , e ) ; } } return accessToken ; }


Input: Get metric value for given rule evaluation . [CODESPLIT] public static Object getMetricValue ( MetricRegistry metrics , String metricId , MetricType metricType , MetricElement metricElement ) throws ObserverException { // We moved the logic of CURRENT_BATCH_AGE and TIME_IN_CURRENT_STAGE due to multi-threaded framework if ( metricElement . isOneOf ( MetricElement . CURRENT_BATCH_AGE , MetricElement . TIME_IN_CURRENT_STAGE ) ) { switch ( metricElement ) { case CURRENT_BATCH_AGE : return getTimeFromRunner ( metrics , PipeRunner . METRIC_BATCH_START_TIME ) ; case TIME_IN_CURRENT_STAGE : return getTimeFromRunner ( metrics , PipeRunner . METRIC_STAGE_START_TIME ) ; default : throw new IllegalStateException ( Utils . format ( "Unknown metric type '{}'" , metricType ) ) ; } } // Default path Metric metric = getMetric ( metrics , metricId , metricType ) ; if ( metric != null ) { return getMetricValue ( metricElement , metricType , metric ) ; } return null ; }


Input: Return calculated metric - from all the runners that are available for given pipeline return the biggest difference between given metric and System . currentTimeMillis () . The semantic is that the runner metric stores start time of certain events ( batch start time stage start time ) and we need to find out what is the longer running time for one of those metrics . [CODESPLIT] private static long getTimeFromRunner ( MetricRegistry metrics , String runnerMetricName ) { // First get number of total runners from the runtime gauge RuntimeStats runtimeStats = ( RuntimeStats ) ( ( Gauge ) getMetric ( metrics , "RuntimeStatsGauge.gauge" , MetricType . GAUGE ) ) . getValue ( ) ; long totalRunners = runtimeStats . getTotalRunners ( ) ; long currentTime = System . currentTimeMillis ( ) ; long maxTime = 0 ; // Then iterate over all runners and find the biggest time difference for ( int runnerId = 0 ; runnerId < totalRunners ; runnerId ++ ) { Map < String , Object > runnerMetrics = ( Map < String , Object > ) ( ( Gauge ) getMetric ( metrics , "runner." + runnerId , MetricType . GAUGE ) ) . getValue ( ) ; // Get current value long value = ( long ) runnerMetrics . getOrDefault ( runnerMetricName , 0L ) ; // Zero means that the runner is not in use at all and thus calculating running time makes no sense if ( value == 0 ) { continue ; } long runTime = currentTime - value ; if ( maxTime < runTime ) { maxTime = runTime ; } } return maxTime ; }


Input: Starts as a standalone file server and waits for Enter . [CODESPLIT] public static void main ( String [ ] args ) { // Defaults int port = 8080 ; String host = null ; // bind to all interfaces by default List < File > rootDirs = new ArrayList < File > ( ) ; boolean quiet = false ; String cors = null ; Map < String , String > options = new HashMap < String , String > ( ) ; // Parse command-line, with short and long versions of the options. for ( int i = 0 ; i < args . length ; ++ i ) { if ( "-h" . equalsIgnoreCase ( args [ i ] ) || "--host" . equalsIgnoreCase ( args [ i ] ) ) { host = args [ i + 1 ] ; } else if ( "-p" . equalsIgnoreCase ( args [ i ] ) || "--port" . equalsIgnoreCase ( args [ i ] ) ) { port = Integer . parseInt ( args [ i + 1 ] ) ; } else if ( "-q" . equalsIgnoreCase ( args [ i ] ) || "--quiet" . equalsIgnoreCase ( args [ i ] ) ) { quiet = true ; } else if ( "-d" . equalsIgnoreCase ( args [ i ] ) || "--dir" . equalsIgnoreCase ( args [ i ] ) ) { rootDirs . add ( new File ( args [ i + 1 ] ) . getAbsoluteFile ( ) ) ; } else if ( args [ i ] . startsWith ( "--cors" ) ) { cors = "*" ; int equalIdx = args [ i ] . indexOf ( ' ' ) ; if ( equalIdx > 0 ) { cors = args [ i ] . substring ( equalIdx + 1 ) ; } } else if ( "--licence" . equalsIgnoreCase ( args [ i ] ) ) { System . out . println ( SimpleWebServer . LICENCE + "\n" ) ; } else if ( args [ i ] . startsWith ( "-X:" ) ) { int dot = args [ i ] . indexOf ( ' ' ) ; if ( dot > 0 ) { String name = args [ i ] . substring ( 0 , dot ) ; String value = args [ i ] . substring ( dot + 1 , args [ i ] . length ( ) ) ; options . put ( name , value ) ; } } } if ( rootDirs . isEmpty ( ) ) { rootDirs . add ( new File ( "." ) . getAbsoluteFile ( ) ) ; } options . put ( "host" , host ) ; options . put ( "port" , "" + port ) ; options . put ( "quiet" , String . valueOf ( quiet ) ) ; StringBuilder sb = new StringBuilder ( ) ; for ( File dir : rootDirs ) { if ( sb . length ( ) > 0 ) { sb . append ( ":" ) ; } try { sb . append ( dir . getCanonicalPath ( ) ) ; } catch ( IOException ignored ) { } } options . put ( "home" , sb . toString ( ) ) ; ServiceLoader < WebServerPluginInfo > serviceLoader = ServiceLoader . load ( WebServerPluginInfo . class ) ; for ( WebServerPluginInfo info : serviceLoader ) { String [ ] mimeTypes = info . getMimeTypes ( ) ; for ( String mime : mimeTypes ) { String [ ] indexFiles = info . getIndexFilesForMimeType ( mime ) ; if ( ! quiet ) { System . out . print ( "# Found plugin for Mime type: \"" + mime + "\"" ) ; if ( indexFiles != null ) { System . out . print ( " (serving index files: " ) ; for ( String indexFile : indexFiles ) { System . out . print ( indexFile + " " ) ; } } System . out . println ( ")." ) ; } registerPluginForMimeType ( indexFiles , mime , info . getWebServerPlugin ( mime ) , options ) ; } } ServerRunner . executeInstance ( new SimpleWebServer ( host , port , rootDirs , quiet , cors ) ) ; }


Input: Serves file from homeDir and its subdirectories ( only ) . Uses only URI ignores all headers and HTTP parameters . [CODESPLIT] Response serveFile ( String uri , Map < String , String > header , File file , String mime ) { Response res ; try { // Calculate etag String etag = Integer . toHexString ( ( file . getAbsolutePath ( ) + file . lastModified ( ) + "" + file . length ( ) ) . hashCode ( ) ) ; // Support (simple) skipping: long startFrom = 0 ; long endAt = - 1 ; String range = header . get ( "range" ) ; if ( range != null ) { if ( range . startsWith ( "bytes=" ) ) { range = range . substring ( "bytes=" . length ( ) ) ; int minus = range . indexOf ( ' ' ) ; try { if ( minus > 0 ) { startFrom = Long . parseLong ( range . substring ( 0 , minus ) ) ; endAt = Long . parseLong ( range . substring ( minus + 1 ) ) ; } } catch ( NumberFormatException ignored ) { } } } // get if-range header. If present, it must match etag or else we // should ignore the range request String ifRange = header . get ( "if-range" ) ; boolean headerIfRangeMissingOrMatching = ( ifRange == null || etag . equals ( ifRange ) ) ; String ifNoneMatch = header . get ( "if-none-match" ) ; boolean headerIfNoneMatchPresentAndMatching = ifNoneMatch != null && ( "*" . equals ( ifNoneMatch ) || ifNoneMatch . equals ( etag ) ) ; // Change return code and add Content-Range header when skipping is // requested long fileLen = file . length ( ) ; if ( headerIfRangeMissingOrMatching && range != null && startFrom >= 0 && startFrom < fileLen ) { // range request that matches current etag // and the startFrom of the range is satisfiable if ( headerIfNoneMatchPresentAndMatching ) { // range request that matches current etag // and the startFrom of the range is satisfiable // would return range from file // respond with not-modified res = newFixedLengthResponse ( Status . NOT_MODIFIED , mime , "" ) ; res . addHeader ( "ETag" , etag ) ; } else { if ( endAt < 0 ) { endAt = fileLen - 1 ; } long newLen = endAt - startFrom + 1 ; if ( newLen < 0 ) { newLen = 0 ; } FileInputStream fis = new FileInputStream ( file ) ; fis . skip ( startFrom ) ; res = Response . newFixedLengthResponse ( Status . PARTIAL_CONTENT , mime , fis , newLen ) ; res . addHeader ( "Accept-Ranges" , "bytes" ) ; res . addHeader ( "Content-Length" , "" + newLen ) ; res . addHeader ( "Content-Range" , "bytes " + startFrom + "-" + endAt + "/" + fileLen ) ; res . addHeader ( "ETag" , etag ) ; } } else { if ( headerIfRangeMissingOrMatching && range != null && startFrom >= fileLen ) { // return the size of the file // 4xx responses are not trumped by if-none-match res = newFixedLengthResponse ( Status . RANGE_NOT_SATISFIABLE , NanoHTTPD . MIME_PLAINTEXT , "" ) ; res . addHeader ( "Content-Range" , "bytes */" + fileLen ) ; res . addHeader ( "ETag" , etag ) ; } else if ( range == null && headerIfNoneMatchPresentAndMatching ) { // full-file-fetch request // would return entire file // respond with not-modified res = newFixedLengthResponse ( Status . NOT_MODIFIED , mime , "" ) ; res . addHeader ( "ETag" , etag ) ; } else if ( ! headerIfRangeMissingOrMatching && headerIfNoneMatchPresentAndMatching ) { // range request that doesn't match current etag // would return entire (different) file // respond with not-modified res = newFixedLengthResponse ( Status . NOT_MODIFIED , mime , "" ) ; res . addHeader ( "ETag" , etag ) ; } else { // supply the file res = newFixedFileResponse ( file , mime ) ; res . addHeader ( "Content-Length" , "" + fileLen ) ; res . addHeader ( "ETag" , etag ) ; } } } catch ( IOException ioe ) { res = getForbiddenResponse ( "Reading file failed." ) ; } return res ; }


Input: Decodes the sent headers and loads the data into Key / value pairs [CODESPLIT] private void decodeHeader ( BufferedReader in , Map < String , String > pre , Map < String , List < String > > parms , Map < String , String > headers ) throws ResponseException { try { // Read the request line String inLine = in . readLine ( ) ; if ( inLine == null ) { return ; } StringTokenizer st = new StringTokenizer ( inLine ) ; if ( ! st . hasMoreTokens ( ) ) { throw new ResponseException ( Status . BAD_REQUEST , "BAD REQUEST: Syntax error. Usage: GET /example/file.html" ) ; } pre . put ( "method" , st . nextToken ( ) ) ; if ( ! st . hasMoreTokens ( ) ) { throw new ResponseException ( Status . BAD_REQUEST , "BAD REQUEST: Missing URI. Usage: GET /example/file.html" ) ; } String uri = st . nextToken ( ) ; // Decode parameters from the URI int qmi = uri . indexOf ( ' ' ) ; if ( qmi >= 0 ) { decodeParms ( uri . substring ( qmi + 1 ) , parms ) ; uri = NanoHTTPD . decodePercent ( uri . substring ( 0 , qmi ) ) ; } else { uri = NanoHTTPD . decodePercent ( uri ) ; } // If there's another token, its protocol version, // followed by HTTP headers. // NOTE: this now forces header names lower case since they are // case insensitive and vary by client. if ( st . hasMoreTokens ( ) ) { protocolVersion = st . nextToken ( ) ; } else { protocolVersion = "HTTP/1.1" ; NanoHTTPD . LOG . log ( Level . FINE , "no protocol version specified, strange. Assuming HTTP/1.1." ) ; } String line = in . readLine ( ) ; while ( line != null && ! line . trim ( ) . isEmpty ( ) ) { int p = line . indexOf ( ' ' ) ; if ( p >= 0 ) { headers . put ( line . substring ( 0 , p ) . trim ( ) . toLowerCase ( Locale . US ) , line . substring ( p + 1 ) . trim ( ) ) ; } line = in . readLine ( ) ; } pre . put ( "uri" , uri ) ; } catch ( IOException ioe ) { throw new ResponseException ( Status . INTERNAL_ERROR , "SERVER INTERNAL ERROR: IOException: " + ioe . getMessage ( ) , ioe ) ; } }