Input: Parse the given string into Date object . [CODESPLIT] public Date parseDate ( String str ) { try { return dateFormat . parse ( str ) ; } catch ( java . text . ParseException e ) { throw new RuntimeException ( e ) ; } }


Input: Select the Accept header s value from the given accepts array : if JSON exists in the given array use it ; otherwise use all of them ( joining into a string ) [CODESPLIT] public String selectHeaderAccept ( String [ ] accepts ) { if ( accepts . length == 0 ) return null ; if ( StringUtil . containsIgnoreCase ( accepts , "application/json" ) ) return "application/json" ; return StringUtil . join ( accepts , "," ) ; }


Input: Select the Content - Type header s value from the given array : if JSON exists in the given array use it ; otherwise use the first one of the array . [CODESPLIT] public String selectHeaderContentType ( String [ ] contentTypes ) { if ( contentTypes . length == 0 ) return "application/json" ; if ( StringUtil . containsIgnoreCase ( contentTypes , "application/json" ) ) return "application/json" ; return contentTypes [ 0 ] ; }


Input: Escape the given string to be used as URL query value . [CODESPLIT] public String escapeString ( String str ) { try { return URLEncoder . encode ( str , "utf8" ) . replaceAll ( "\\+" , "%20" ) ; } catch ( UnsupportedEncodingException e ) { return str ; } }


Input: Serialize the given Java object into string according the given Content - Type ( only JSON is supported for now ) . [CODESPLIT] public String serialize ( Object obj , String contentType ) throws ApiException { if ( contentType . startsWith ( "application/json" ) ) { return json . serialize ( obj ) ; } else { throw new ApiException ( 400 , "can not serialize object into Content-Type: " + contentType ) ; } }


Input: Deserialize response body to Java object according to the Content - Type . [CODESPLIT] private < T > T deserialize ( Response response , TypeRef returnType ) throws ApiException { String contentType = null ; List < Object > contentTypes = response . getHeaders ( ) . get ( "Content-Type" ) ; if ( contentTypes != null && ! contentTypes . isEmpty ( ) ) { contentType = ( String ) contentTypes . get ( 0 ) ; } if ( contentType == null ) { throw new ApiException ( 500 , "missing Content-Type in response" ) ; } if ( contentType . startsWith ( "application/json" ) ) { String body ; if ( response . hasEntity ( ) ) { body = response . readEntity ( String . class ) ; } else { body = "" ; } if ( body . length ( ) > 0 ) { return json . deserialize ( body , returnType ) ; } return null ; } if ( contentType . startsWith ( "image" ) ) { return ( T ) response . readEntity ( InputStream . class ) ; } else { throw new ApiException ( 500 , "can not deserialize Content-Type: " + contentType ) ; } }


Input: Get an existing client or create a new client to handle HTTP request . [CODESPLIT] private Client getClient ( ) { if ( ! hostMap . containsKey ( basePath ) ) { ClientConfig config = new ClientConfig ( ) ; config . property ( ClientProperties . SUPPRESS_HTTP_COMPLIANCE_VALIDATION , true ) ; Client client = ClientBuilder . newClient ( config ) ; client . register ( new CsrfProtectionFilter ( "CSRF" ) ) ; hostMap . put ( basePath , client ) ; } return hostMap . get ( basePath ) ; }


Input: Connect to the database [CODESPLIT] private GPUdb initConnection ( KineticaConfigBean conf ) throws GPUdbException , StageException { KineticaConnectionUtils kineticaConnectionUtils = new KineticaConnectionUtils ( ) ; return kineticaConnectionUtils . getGPUdb ( conf ) ; }


Input: Get metadata for the table [CODESPLIT] private void getTableMetadata ( GPUdb gpudb , String tableName ) throws GPUdbException { KineticaTableUtils kineticaTableUtils = new KineticaTableUtils ( gpudb , tableName ) ; type = kineticaTableUtils . getType ( ) ; }


Input: Create a BulkInserter [CODESPLIT] private BulkInserter < IndexedRecord > createBulkInserter ( GPUdb gpudb , Type type , KineticaConfigBean conf ) throws GPUdbException { KineticaBulkInserterUtils kineticaBulkInserterUtils = new KineticaBulkInserterUtils ( gpudb , type , conf ) ; return kineticaBulkInserterUtils . createBulkInserter ( ) ; }


Input: Create new event record according for this stage context and event context . [CODESPLIT] public EventBuilder create ( Stage . Context context , ToEventContext toEvent ) { return new EventBuilder ( context , toEvent ) ; }


Input: Build a schema with type record . This will be the top level schema and contains fields [CODESPLIT] public static Schema buildSchema ( Map < String , Schema > fields , Object ... levels ) { List < Schema . Field > recordFields = new ArrayList <> ( fields . size ( ) ) ; for ( Map . Entry < String , Schema > entry : fields . entrySet ( ) ) { recordFields . add ( new Schema . Field ( entry . getKey ( ) , entry . getValue ( ) , null , // Avro's Schema.Field constructor requires doc. entry . getValue ( ) . getJsonProp ( "default" ) ) ) ; } Schema recordSchema ; if ( levels . length == 0 ) { recordSchema = Schema . createRecord ( schemaName , null , null , false ) ; } else { LinkedList < String > lvl = ( LinkedList < String > ) levels [ 0 ] ; recordSchema = Schema . createRecord ( joiner . join ( lvl ) , null , null , false ) ; } recordSchema . setFields ( recordFields ) ; return recordSchema ; }


Input: This is called when JDBC target didn t find sdc . operation . code in record header but found oracle . cdc . operation . Since oracle . cdc . operation contains Oracle specific operation code we need to convert to SDC operation code . [CODESPLIT] public static int convertFromOracleToSDCCode ( String code ) { try { int intCode = Integer . parseInt ( code ) ; switch ( intCode ) { case INSERT_CODE : return OperationType . INSERT_CODE ; case DELETE_CODE : return OperationType . DELETE_CODE ; case UPDATE_CODE : case SELECT_FOR_UPDATE_CODE : return OperationType . UPDATE_CODE ; default : //DDL_CODE throw new UnsupportedOperationException ( Utils . format ( "Operation code {} is not supported" , code ) ) ; } } catch ( NumberFormatException ex ) { throw new NumberFormatException ( "Operation code must be a numeric value. " + ex . getMessage ( ) ) ; } }


Input: Add jars containing the following classes to the job s classpath . [CODESPLIT] public static void addJarsToJob ( Configuration conf , Class ... klasses ) { // Build set of jars that needs to be added, order doesn't matter for us and we will remove duplicates Set < String > additinonalJars = new HashSet <> ( ) ; for ( Class klass : klasses ) { final String jar = jarForClass ( klass ) ; LOG . info ( "Adding jar {} for class {}" , jar , klass . getCanonicalName ( ) ) ; additinonalJars . add ( jar ) ; } appendJars ( conf , additinonalJars ) ; }


Input: Add jars whose names contain the given patterns to the job s classpath . [CODESPLIT] public static void addJarsToJob ( Configuration conf , boolean allowMultiple , String ... jarPatterns ) { final ClassLoader loader = MapreduceUtils . class . getClassLoader ( ) ; if ( ! ( loader instanceof URLClassLoader ) ) { throw new IllegalStateException ( String . format ( "ClassLoader for %s is not an instance of URLClassLoader (it is %s), and thus this method cannot be used" , MapreduceUtils . class . getCanonicalName ( ) , loader . getClass ( ) . getCanonicalName ( ) ) ) ; } final URLClassLoader urlClassLoader = ( URLClassLoader ) loader ; addJarsToJob ( conf , allowMultiple , urlClassLoader . getURLs ( ) , jarPatterns ) ; }


Input: { [CODESPLIT] @ Override public void initialize ( InitializationInput initializationInput ) { shardId = initializationInput . getShardId ( ) ; if ( LOG . isDebugEnabled ( ) ) { LOG . debug ( "Initializing record processor at: {}" , initializationInput . getExtendedSequenceNumber ( ) . toString ( ) ) ; LOG . debug ( "Initializing record processor for shard: {}" , shardId ) ; } }


Input: { [CODESPLIT] @ Override public void processRecords ( ProcessRecordsInput processRecordsInput ) { LOG . debug ( "RecordProcessor processRecords called" ) ; try { IRecordProcessorCheckpointer checkpointer = processRecordsInput . getCheckpointer ( ) ; startBatch ( ) ; Optional < Record > lastProcessedRecord = Optional . empty ( ) ; int recordCount = 0 ; for ( Record kRecord : processRecordsInput . getRecords ( ) ) { try { KinesisUtil . processKinesisRecord ( shardId , kRecord , parserFactory ) . forEach ( batchMaker :: addRecord ) ; lastProcessedRecord = Optional . of ( kRecord ) ; if ( ++ recordCount == maxBatchSize ) { recordCount = 0 ; finishBatch ( checkpointer , kRecord ) ; startBatch ( ) ; } } catch ( DataParserException | IOException e ) { com . streamsets . pipeline . api . Record record = context . createRecord ( kRecord . getSequenceNumber ( ) ) ; record . set ( Field . create ( kRecord . getData ( ) . array ( ) ) ) ; try { errorRecordHandler . onError ( new OnRecordErrorException ( record , Errors . KINESIS_03 , kRecord . getSequenceNumber ( ) , e . toString ( ) , e ) ) ; // move the lastProcessedRecord forward if not set to stop pipeline lastProcessedRecord = Optional . of ( kRecord ) ; } catch ( StageException ex ) { // KCL skips over the data records that were passed prior to the exception // that is, these records are not re-sent to this record processor // or to any other record processor in the consumer. lastProcessedRecord . ifPresent ( r -> finishBatch ( checkpointer , r ) ) ; try { error . put ( ex ) ; } catch ( InterruptedException ie ) { Thread . currentThread ( ) . interrupt ( ) ; } return ; } } } lastProcessedRecord . ifPresent ( r -> finishBatch ( checkpointer , r ) ) ; } catch ( Exception e ) { LOG . error ( "Unknown error while processing records: {}" , e . toString ( ) , e ) ; context . reportError ( Errors . KINESIS_17 , e . toString ( ) ) ; } }


Input: We don t checkpoint on SHUTDOWN_REQUESTED because we currently always checkpoint each batch in { @link #processRecords } . [CODESPLIT] @ Override public void shutdown ( ShutdownInput shutdownInput ) { LOG . info ( "Shutting down record processor for shard: {}" , shardId ) ; if ( ShutdownReason . TERMINATE . equals ( shutdownInput . getShutdownReason ( ) ) ) { // Shard is closed / finished processing. Checkpoint all processing up to here. try { shutdownInput . getCheckpointer ( ) . checkpoint ( ) ; LOG . debug ( "Checkpointed due to record processor shutdown request." ) ; } catch ( InvalidStateException | ShutdownException e ) { LOG . error ( "Error checkpointing batch: {}" , e . toString ( ) , e ) ; } } }


Input: { [CODESPLIT] @ Override public void process ( final HttpRequest request , final HttpContext context ) throws HttpException , IOException { URIBuilder uriBuilder ; try { uriBuilder = new URIBuilder ( request . getRequestLine ( ) . getUri ( ) ) ; } catch ( URISyntaxException e ) { throw new IOException ( "Invalid URI" , e ) ; } // Copy Apache HttpRequest to AWS DefaultRequest DefaultRequest < ? > signableRequest = new DefaultRequest <> ( service ) ; HttpHost host = ( HttpHost ) context . getAttribute ( HttpCoreContext . HTTP_TARGET_HOST ) ; if ( host != null ) { signableRequest . setEndpoint ( URI . create ( host . toURI ( ) ) ) ; } final HttpMethodName httpMethod = HttpMethodName . fromValue ( request . getRequestLine ( ) . getMethod ( ) ) ; signableRequest . setHttpMethod ( httpMethod ) ; try { signableRequest . setResourcePath ( uriBuilder . build ( ) . getRawPath ( ) ) ; } catch ( URISyntaxException e ) { throw new IOException ( "Invalid URI" , e ) ; } if ( request instanceof HttpEntityEnclosingRequest ) { HttpEntityEnclosingRequest httpEntityEnclosingRequest = ( HttpEntityEnclosingRequest ) request ; if ( httpEntityEnclosingRequest . getEntity ( ) != null ) { signableRequest . setContent ( httpEntityEnclosingRequest . getEntity ( ) . getContent ( ) ) ; } } signableRequest . setParameters ( nvpToMapParams ( uriBuilder . getQueryParams ( ) ) ) ; signableRequest . setHeaders ( headerArrayToMap ( request . getAllHeaders ( ) ) ) ; // Sign it signer . sign ( signableRequest , awsCredentialsProvider . getCredentials ( ) ) ; // Now copy everything back request . setHeaders ( mapToHeaderArray ( signableRequest . getHeaders ( ) ) ) ; if ( request instanceof HttpEntityEnclosingRequest ) { HttpEntityEnclosingRequest httpEntityEnclosingRequest = ( HttpEntityEnclosingRequest ) request ; if ( httpEntityEnclosingRequest . getEntity ( ) != null ) { BasicHttpEntity basicHttpEntity = new BasicHttpEntity ( ) ; basicHttpEntity . setContent ( signableRequest . getContent ( ) ) ; httpEntityEnclosingRequest . setEntity ( basicHttpEntity ) ; } } }


Input: Returns true if the first ORDER BY field matches fieldName [CODESPLIT] private boolean checkFieldOrderByList ( SOQLParser . FieldOrderByListContext fieldOrderByList , String fieldName ) { return fieldOrderByList . fieldOrderByElement ( 0 ) . fieldElement ( ) . getText ( ) . equalsIgnoreCase ( fieldName ) ; }


Input: Returns true if any of the nested conditions contains fieldName [CODESPLIT] private boolean checkConditionExpressions ( SOQLParser . ConditionExpressionsContext conditionExpressions , String fieldName ) { for ( SOQLParser . ConditionExpressionContext ce : conditionExpressions . conditionExpression ( ) ) { if ( ( ce . conditionExpressions ( ) != null && checkConditionExpressions ( ce . conditionExpressions ( ) , fieldName ) ) || ( ce . fieldExpression ( ) != null && ce . fieldExpression ( ) . fieldElement ( ) . getText ( ) . equalsIgnoreCase ( fieldName ) ) ) { return true ; } } return false ; }


Input: { [CODESPLIT] @ Override public void destroy ( ) { // SDC-6258 - destroy() is called from a different thread, so we // need to signal produce() to terminate early destroyed . set ( true ) ; if ( job != null ) { try { try { bulkConnection . abortJob ( job . getId ( ) ) ; } catch ( AsyncApiException e ) { ForceUtils . renewSession ( bulkConnection , e ) ; bulkConnection . abortJob ( job . getId ( ) ) ; } job = null ; } catch ( AsyncApiException e ) { LOG . error ( "Exception while aborting job" , e ) ; } } job = null ; if ( forceConsumer != null ) { try { forceConsumer . stop ( ) ; forceConsumer = null ; } catch ( Exception e ) { LOG . error ( "Exception while stopping ForceStreamConsumer." , e ) ; } if ( ! messageQueue . isEmpty ( ) ) { LOG . error ( "Queue still had {} entities at shutdown." , messageQueue . size ( ) ) ; } else { LOG . info ( "Queue was empty at shutdown. No data lost." ) ; } } if ( recordCreator != null ) { recordCreator . destroy ( ) ; } // Clean up any open resources. super . destroy ( ) ; }


Input: { [CODESPLIT] @ Override public String produce ( String lastSourceOffset , int maxBatchSize , BatchMaker batchMaker ) throws StageException { String nextSourceOffset = null ; LOG . debug ( "lastSourceOffset: {}" , lastSourceOffset ) ; // send event only once for each time we run out of data. if ( shouldSendNoMoreDataEvent ) { CommonEvents . NO_MORE_DATA . create ( getContext ( ) ) . createAndSend ( ) ; shouldSendNoMoreDataEvent = false ; return lastSourceOffset ; } int batchSize = Math . min ( conf . basicConfig . maxBatchSize , maxBatchSize ) ; if ( ! conf . queryExistingData || ( null != lastSourceOffset && lastSourceOffset . startsWith ( EVENT_ID_OFFSET_PREFIX ) ) ) { if ( conf . subscribeToStreaming ) { nextSourceOffset = streamingProduce ( lastSourceOffset , batchSize , batchMaker ) ; } else { // We're done reading existing data, but we don't want to subscribe to Streaming API return null ; } } else if ( conf . queryExistingData ) { if ( ! queryInProgress ( ) ) { long now = System . currentTimeMillis ( ) ; long delay = Math . max ( 0 , ( lastQueryCompletedTime + ( 1000 * conf . queryInterval ) ) - now ) ; if ( delay > 0 ) { // Sleep in one second increments so we don't tie up the app. LOG . info ( "{}ms remaining until next fetch." , delay ) ; ThreadUtil . sleep ( Math . min ( delay , 1000 ) ) ; return lastSourceOffset ; } } if ( conf . useBulkAPI ) { nextSourceOffset = bulkProduce ( lastSourceOffset , batchSize , batchMaker ) ; } else { nextSourceOffset = soapProduce ( lastSourceOffset , batchSize , batchMaker ) ; } } else if ( conf . subscribeToStreaming ) { // No offset, and we're not querying existing data, so switch to streaming nextSourceOffset = READ_EVENTS_FROM_NOW ; } LOG . debug ( "nextSourceOffset: {}" , nextSourceOffset ) ; return nextSourceOffset ; }


Input: If passed a valid fileOffsetString it will return what is the offset lag in the file . [CODESPLIT] public static long getOffsetLagForFile ( String fileOffsetString ) throws IOException { long offset = FileContextProviderUtil . getLongOffsetFromFileOffset ( fileOffsetString ) ; //We are refreshing the live file here because we are going to get the size by using path. LiveFile file = FileContextProviderUtil . getRefreshedLiveFileFromFileOffset ( fileOffsetString ) ; long fileSizeInBytes = Files . size ( file . getPath ( ) . toAbsolutePath ( ) ) ; return ( fileSizeInBytes - offset ) ; }


Input: Visible due to JVM requirements only [CODESPLIT] public static void premain ( String args , Instrumentation instrumentation ) { if ( BootstrapMain . instrumentation == null ) { BootstrapMain . instrumentation = instrumentation ; } else { throw new IllegalStateException ( "Premain method cannot be called twice (" + BootstrapMain . instrumentation + ")" ) ; } }


Input: if whitelist is * set is NULL else whitelist has the whitelisted values [CODESPLIT] public static Set < String > getWhiteList ( String configDir , String property ) { Set < String > set = null ; File whiteListFile = new File ( configDir , WHITE_LIST_FILE ) . getAbsoluteFile ( ) ; if ( whiteListFile . exists ( ) ) { try ( InputStream is = new FileInputStream ( whiteListFile ) ) { Properties props = new Properties ( ) ; props . load ( is ) ; String whiteList = props . getProperty ( property ) ; if ( whiteList == null ) { throw new IllegalArgumentException ( String . format ( WHITE_LIST_PROPERTY_MISSING_MSG , property , whiteListFile ) ) ; } whiteList = whiteList . trim ( ) ; if ( ! whiteList . equals ( ALL_VALUES ) ) { set = new HashSet <> ( ) ; for ( String name : whiteList . split ( "," ) ) { name = name . trim ( ) ; if ( ! name . isEmpty ( ) ) { set . add ( "+" + name . trim ( ) ) ; } } } } catch ( IOException ex ) { throw new IllegalArgumentException ( String . format ( WHITE_LIST_COULD_NOT_LOAD_FILE_MSG , whiteListFile , ex . toString ( ) ) , ex ) ; } } return set ; }


Input: Visible for testing [CODESPLIT] public static Map < String , List < URL > > getStageLibrariesClasspaths ( String stageLibrariesDir , String librariesExtraDir , final Set < String > stageLibs , String libsCommonLibDir ) throws Exception { Map < String , List < URL > > map = new LinkedHashMap < String , List < URL > > ( ) ; File baseDir = new File ( stageLibrariesDir ) . getAbsoluteFile ( ) ; if ( baseDir . exists ( ) ) { File [ ] libDirs = baseDir . listFiles ( createStageLibFilter ( stageLibs ) ) ; StringBuilder commonLibJars = new StringBuilder ( ) ; if ( libsCommonLibDir != null ) { commonLibJars . append ( new File ( libsCommonLibDir ) . getAbsolutePath ( ) ) . append ( FILE_SEPARATOR ) . append ( JARS_WILDCARD ) . append ( CLASSPATH_SEPARATOR ) ; } for ( File libDir : libDirs ) { File jarsDir = new File ( libDir , STAGE_LIB_JARS_DIR ) ; File etc = new File ( libDir , STAGE_LIB_CONF_DIR ) ; if ( ! jarsDir . exists ( ) ) { throw new IllegalArgumentException ( String . format ( MISSING_STAGE_LIB_JARS_DIR_MSG , libDir ) ) ; } StringBuilder sb = new StringBuilder ( ) ; if ( etc . exists ( ) ) { sb . append ( etc . getAbsolutePath ( ) ) . append ( FILE_SEPARATOR ) . append ( CLASSPATH_SEPARATOR ) ; } sb . append ( commonLibJars ) ; sb . append ( jarsDir . getAbsolutePath ( ) ) . append ( FILE_SEPARATOR ) . append ( JARS_WILDCARD ) ; // add extralibs if avail if ( librariesExtraDir != null ) { File libExtraDir = new File ( librariesExtraDir , libDir . getName ( ) ) ; if ( libExtraDir . exists ( ) ) { File extraJarsDir = new File ( libExtraDir , STAGE_LIB_JARS_DIR ) ; if ( extraJarsDir . exists ( ) ) { sb . append ( CLASSPATH_SEPARATOR ) . append ( extraJarsDir . getAbsolutePath ( ) ) . append ( FILE_SEPARATOR ) . append ( JARS_WILDCARD ) ; } File extraEtc = new File ( libExtraDir , STAGE_LIB_CONF_DIR ) ; if ( extraEtc . exists ( ) ) { sb . append ( CLASSPATH_SEPARATOR ) . append ( extraEtc . getAbsolutePath ( ) ) ; } } } map . put ( libDir . getParentFile ( ) . getName ( ) + FILE_SEPARATOR + libDir . getName ( ) , getClasspathUrls ( sb . toString ( ) ) ) ; } } else { throw new IllegalArgumentException ( String . format ( MISSING_STAGE_LIBRARIES_DIR_MSG , baseDir ) ) ; } return map ; }


Input: Visible for testing [CODESPLIT] public static List < URL > getClasspathUrls ( String classPath ) throws Exception { List < URL > urls = new ArrayList < URL > ( ) ; for ( String path : classPath . split ( CLASSPATH_SEPARATOR ) ) { if ( ! path . isEmpty ( ) ) { if ( path . toLowerCase ( ) . endsWith ( JARS_WILDCARD ) ) { path = path . substring ( 0 , path . length ( ) - JARS_WILDCARD . length ( ) ) ; File f = new File ( path ) . getAbsoluteFile ( ) ; if ( f . exists ( ) ) { File [ ] jars = f . listFiles ( new FileFilter ( ) { @ Override public boolean accept ( File pathname ) { return pathname . getName ( ) . toLowerCase ( ) . endsWith ( JARS_WILDCARD . substring ( 1 ) ) ; } } ) ; for ( File jar : jars ) { urls . add ( jar . toURI ( ) . toURL ( ) ) ; } } else { throw new IllegalArgumentException ( String . format ( CLASSPATH_DIR_DOES_NOT_EXIST_MSG , f ) ) ; } } else { if ( ! path . endsWith ( FILE_SEPARATOR ) ) { path = path + FILE_SEPARATOR ; } File f = new File ( path ) . getAbsoluteFile ( ) ; if ( f . exists ( ) ) { if ( f . isDirectory ( ) ) { urls . add ( f . toURI ( ) . toURL ( ) ) ; } else { throw new IllegalArgumentException ( String . format ( CLASSPATH_PATH_S_IS_NOT_A_DIR_MSG , f ) ) ; } } else { throw new IllegalArgumentException ( String . format ( CLASSPATH_DIR_DOES_NOT_EXIST_MSG , f ) ) ; } } } } return urls ; }


Input: This API is being used by ClusterKafkaSource [CODESPLIT] public int getParallelism ( ) throws StageException { if ( originParallelism == 0 ) { //origin parallelism is not yet calculated originParallelism = kafkaValidationUtil . getPartitionCount ( conf . metadataBrokerList , conf . topic , new HashMap < String , Object > ( conf . kafkaConsumerConfigs ) , 3 , 1000 ) ; if ( originParallelism < 1 ) { throw new StageException ( KafkaErrors . KAFKA_42 , conf . topic ) ; } } return originParallelism ; }


Input: Arranges the urls in the following order : <ul > <li > stage lib jars< / li > <li > protolib jars< / li > <li > non protolib jars< / li > < / ul > [CODESPLIT] static List < URL > bringStageAndProtoLibsToFront ( String stageLibName , List < URL > urls ) { List < URL > otherJars = new ArrayList <> ( ) ; List < URL > protolibJars = new ArrayList <> ( ) ; List < URL > stageLibjars = new ArrayList <> ( ) ; for ( URL url : urls ) { String str = url . toExternalForm ( ) ; if ( str . endsWith ( ".jar" ) ) { int nameIdx = str . lastIndexOf ( "/" ) ; if ( nameIdx > - 1 ) { String jarName = str . substring ( nameIdx + 1 ) ; if ( jarName . contains ( "-protolib-" ) ) { // adding only protolib jars protolibJars . add ( url ) ; } else if ( jarName . contains ( stageLibName ) ) { stageLibjars . add ( url ) ; } else { otherJars . add ( url ) ; } } else { otherJars . add ( url ) ; } } else { otherJars . add ( url ) ; } } List < URL > allJars = new ArrayList <> ( ) ; if ( stageLibjars . size ( ) != 1 ) { throw new ExceptionInInitializerError ( "Expected exactly 1 stage lib jar but found " + stageLibjars . size ( ) + " with name " + stageLibName ) ; } allJars . addAll ( stageLibjars ) ; allJars . addAll ( protolibJars ) ; allJars . addAll ( otherJars ) ; return allJars ; }


Input: Executes a query request and returns the results . A timeout is required to avoid waiting for an indeterminate amount of time . If the query fails to complete within the timeout it is aborted . [CODESPLIT] public TableResult runQuery ( QueryJobConfiguration queryConfig , long timeout , long pageSize ) throws StageException { checkArgument ( timeout >= 1000 , "Timeout must be at least one second." ) ; Instant maxTime = Instant . now ( ) . plusMillis ( timeout ) ; // Create a job ID so that we can safely retry. JobId jobId = JobId . of ( UUID . randomUUID ( ) . toString ( ) ) ; JobInfo jobInfo = JobInfo . newBuilder ( queryConfig ) . setJobId ( jobId ) . build ( ) ; Job queryJob = bigquery . create ( jobInfo ) ; // Check for errors if ( queryJob == null ) { LOG . error ( "Job no longer exists: {}" , jobInfo ) ; throw new RuntimeException ( "Job no longer exists: " + jobInfo ) ; } else if ( queryJob . getStatus ( ) . getError ( ) != null ) { BigQueryError error = queryJob . getStatus ( ) . getError ( ) ; LOG . error ( "Query Job execution error: {}" , error ) ; throw new StageException ( Errors . BIGQUERY_02 , error ) ; } //Should consider using .waitFor(RetryOption.totalTimeout()) while ( ! queryJob . isDone ( ) ) { if ( Instant . now ( clock ) . isAfter ( maxTime ) || ! ThreadUtil . sleep ( 100 ) ) { if ( bigquery . cancel ( queryJob . getJobId ( ) ) ) { LOG . info ( "Job {} cancelled successfully." , queryJob . getJobId ( ) ) ; } else { LOG . warn ( "Job {} not found" , queryJob . getJobId ( ) ) ; } throw new StageException ( Errors . BIGQUERY_00 ) ; } } if ( queryJob . getStatus ( ) . getError ( ) != null ) { String errorMsg = queryJob . getStatus ( ) . getError ( ) . toString ( ) ; throw new StageException ( Errors . BIGQUERY_02 , errorMsg ) ; } // Get the results. TableResult result = null ; try { result = queryJob . getQueryResults ( QueryResultsOption . pageSize ( pageSize ) ) ; } catch ( InterruptedException e ) { String errorMsg = e . getMessage ( ) ; throw new StageException ( Errors . BIGQUERY_02 , errorMsg ) ; } return result ; }


Input: Returns the SDC record { @link Field } type mapped from a BigQuery { @link com . google . cloud . bigquery . Field } type . [CODESPLIT] public Field . Type asRecordFieldType ( com . google . cloud . bigquery . Field field ) { Field . Type type = asRecordFieldTypeFunction . apply ( field ) ; return checkNotNull ( type , Utils . format ( "Unsupported type '{}'" , field . getType ( ) ) ) ; }


Input: Converts a list of BigQuery fields to SDC Record fields . The provided parameters must have matching lengths . [CODESPLIT] public LinkedHashMap < String , Field > fieldsToMap ( // NOSONAR List < com . google . cloud . bigquery . Field > schema , List < FieldValue > values ) { checkState ( schema . size ( ) == values . size ( ) , "Schema '{}' and Values '{}' sizes do not match." , schema . size ( ) , values . size ( ) ) ; LinkedHashMap < String , Field > root = new LinkedHashMap <> ( ) ; for ( int i = 0 ; i < values . size ( ) ; i ++ ) { FieldValue value = values . get ( i ) ; com . google . cloud . bigquery . Field field = schema . get ( i ) ; if ( value . getAttribute ( ) . equals ( FieldValue . Attribute . PRIMITIVE ) ) { root . put ( field . getName ( ) , fromPrimitiveField ( field , value ) ) ; } else if ( value . getAttribute ( ) . equals ( FieldValue . Attribute . RECORD ) ) { root . put ( field . getName ( ) , Field . create ( fieldsToMap ( field . getSubFields ( ) , value . getRecordValue ( ) ) ) ) ; } else if ( value . getAttribute ( ) . equals ( FieldValue . Attribute . REPEATED ) ) { root . put ( field . getName ( ) , Field . create ( fromRepeatedField ( field , value . getRepeatedValue ( ) ) ) ) ; } } return root ; }


Input: Repeated fields are simply fields that may appear more than once . In SDC we will represent them as a list field . For example a repeated field of type RECORD would be a { @link Field . Type#LIST } of { @link Field . Type#LIST_MAP } and a repeated field of type STRING would be a { @link Field . Type#LIST } of { @link Field . Type#STRING } . [CODESPLIT] public List < Field > fromRepeatedField ( com . google . cloud . bigquery . Field schema , List < FieldValue > repeatedValue ) { if ( repeatedValue . isEmpty ( ) ) { return Collections . emptyList ( ) ; } FieldValue . Attribute repeatedFieldType = repeatedValue . get ( 0 ) . getAttribute ( ) ; BiFunction < com . google . cloud . bigquery . Field , FieldValue , Field > transform = transforms . get ( repeatedFieldType ) ; return repeatedValue . stream ( ) . map ( v -> transform . apply ( schema , v ) ) . collect ( Collectors . toList ( ) ) ; }


Input: Get the table description from the ShowTableResponse [CODESPLIT] private List < String > getTableDescription ( ) throws GPUdbException { List < List < String >> descriptions = showTableResponse . getTableDescriptions ( ) ; if ( descriptions == null || descriptions . size ( ) != 1 ) { throw new GPUdbException ( "Error getting description for table " + tableName ) ; } return descriptions . get ( 0 ) ; }


Input: if it is not [CODESPLIT] private void validateTableAcceptsInserts ( ) throws GPUdbException { for ( String s : tableDescription ) { if ( s . equalsIgnoreCase ( "COLLECTION" ) ) { throw new GPUdbException ( "Error: table " + tableName + " is a Collection" ) ; } else if ( s . equalsIgnoreCase ( "VIEW" ) ) { throw new GPUdbException ( "Error: table " + tableName + " is a View" ) ; } else if ( s . equalsIgnoreCase ( "JOIN" ) ) { throw new GPUdbException ( "Error: table " + tableName + " is a Join Table" ) ; } else if ( s . equalsIgnoreCase ( "RESULT_TABLE" ) ) { throw new GPUdbException ( "Error: table " + tableName + " is a Result Table" ) ; } } }


Input: Get the Class for the column type [CODESPLIT] private Class < ? > getColumnType ( JSONObject field ) throws GPUdbException { Class < ? > columnType = null ; // The Avro "type" element might be an array if the type is nullable if ( field . get ( "type" ) instanceof JSONArray ) { JSONArray columnTypes = field . getJSONArray ( "type" ) ; for ( int j = 0 ; j < columnTypes . length ( ) ; j ++ ) { String ct = ( String ) columnTypes . get ( j ) ; if ( ! ct . equals ( "null" ) ) { columnType = getClassForType ( ct ) ; break ; } } } else { columnType = getClassForType ( field . getString ( "type" ) ) ; } if ( columnType == null ) { throw new GPUdbException ( "Error getting column type for field: " + field . toString ( ) ) ; } return columnType ; }


Input: the Avro type field ) [CODESPLIT] private boolean typeIsNullable ( JSONObject field ) throws GPUdbException { if ( field . get ( "type" ) instanceof JSONArray ) { JSONArray columnTypes = field . getJSONArray ( "type" ) ; for ( int j = 0 ; j < columnTypes . length ( ) ; j ++ ) { String ct = ( String ) columnTypes . get ( j ) ; if ( ct . equals ( "null" ) ) { return true ; } } } return false ; }


Input: Get the table s schema as a JSON Object [CODESPLIT] private JSONObject getTableSchema ( String tableName , ShowTableResponse showTableResponse ) throws GPUdbException { List < String > schemas = showTableResponse . getTypeSchemas ( ) ; if ( schemas == null || schemas . size ( ) != 1 ) { throw new GPUdbException ( "Error getting schema for table " + tableName ) ; } return new JSONObject ( schemas . get ( 0 ) ) ; }


Input: Get the table s extended column properties [CODESPLIT] private Map < String , List < String > > getColumnProperties ( String tableName , ShowTableResponse showTableResponse ) throws GPUdbException { List < Map < String , List < String > > > columnPropertiesList = showTableResponse . getProperties ( ) ; if ( columnPropertiesList == null || columnPropertiesList . size ( ) != 1 ) { throw new GPUdbException ( "Error getting properties for table " + tableName ) ; } return columnPropertiesList . get ( 0 ) ; }


Input: Get the Java type for a type name [CODESPLIT] private Class < ? > getClassForType ( String typeName ) throws GPUdbException { typeName = typeName . replace ( " " , "" ) ; if ( typeName . equalsIgnoreCase ( STRING_TYPE_NAME ) ) { return String . class ; } else if ( typeName . equalsIgnoreCase ( LONG_TYPE_NAME ) ) { return Long . class ; } else if ( typeName . equalsIgnoreCase ( INTEGER_TYPE_NAME ) ) { return Integer . class ; } else if ( typeName . equalsIgnoreCase ( FLOAT_TYPE_NAME ) ) { return Float . class ; } else if ( typeName . equalsIgnoreCase ( DOUBLE_TYPE_NAME ) ) { return Double . class ; } else if ( typeName . equalsIgnoreCase ( BYTES_TYPE_NAME ) ) { return ByteBuffer . class ; } else { throw new GPUdbException ( "Error: unknown type '" + typeName + "' in table schema" ) ; } }


Input: initialize and validate configuration options [CODESPLIT] public void init ( Target . Context context , List < Target . ConfigIssue > issues ) { List < Host > hosts = getAerospikeHosts ( issues , connectionString , Groups . AEROSPIKE . getLabel ( ) , "aerospikeBeanConfig.connectionString" , context ) ; ClientPolicy cp = new ClientPolicy ( ) ; try { client = new AerospikeClient ( cp , hosts . toArray ( new Host [ hosts . size ( ) ] ) ) ; int retries = 0 ; while ( ! client . isConnected ( ) && retries <= maxRetries ) { if ( retries > maxRetries ) { issues . add ( context . createConfigIssue ( Groups . AEROSPIKE . getLabel ( ) , "aerospikeBeanConfig.connectionString" , AerospikeErrors . AEROSPIKE_03 , connectionString ) ) ; return ; } retries ++ ; try { Thread . sleep ( 100 ) ; } catch ( InterruptedException ignored ) { } } } catch ( AerospikeException ex ) { issues . add ( context . createConfigIssue ( Groups . AEROSPIKE . getLabel ( ) , "aerospikeBeanConfig.connectionString" , AerospikeErrors . AEROSPIKE_03 , connectionString ) ) ; } }


Input: Returns the topic given the record . [CODESPLIT] String getTopic ( Record record ) throws StageException { String result = publisherConf . topic ; if ( publisherConf . runtimeTopicResolution ) { RecordEL . setRecordInContext ( topicVars , record ) ; try { result = topicEval . eval ( topicVars , publisherConf . topicExpression , String . class ) ; if ( isEmpty ( result ) ) { throw new StageException ( Errors . MQTT_08 , publisherConf . topicExpression , record . getHeader ( ) . getSourceId ( ) ) ; } if ( ! allowedTopics . contains ( result ) && ! allowAllTopics ) { throw new StageException ( Errors . MQTT_09 , result , record . getHeader ( ) . getSourceId ( ) ) ; } } catch ( ELEvalException e ) { throw new StageException ( Errors . MQTT_10 , publisherConf . topicExpression , record . getHeader ( ) . getSourceId ( ) , e . toString ( ) ) ; } } return result ; }


Input: Search the provided pattern and get the C standard Date / Time formatting rules and convert them to the Java equivalent . [CODESPLIT] public String convertDateFormat ( String pattern ) { boolean inside = false ; boolean mark = false ; boolean modifiedCommand = false ; StringBuilder buf = new StringBuilder ( ) ; for ( int i = 0 ; i < pattern . length ( ) ; i ++ ) { char c = pattern . charAt ( i ) ; if ( c == ' ' && ! mark ) { mark = true ; } else { if ( mark ) { if ( modifiedCommand ) { //don't do anything--we just wanted to skip a char modifiedCommand = false ; mark = false ; } else { inside = translateCommand ( buf , pattern , i , inside ) ; //It's a modifier code if ( c == ' ' || c == ' ' ) { modifiedCommand = true ; } else { mark = false ; } } } else { if ( ! inside && c != ' ' ) { //We start a literal, which we need to quote buf . append ( "'" ) ; inside = true ; } buf . append ( c ) ; } } } if ( buf . length ( ) > 0 ) { char lastChar = buf . charAt ( buf . length ( ) - 1 ) ; if ( lastChar != ' ' && inside ) { buf . append ( ' ' ) ; } } return buf . toString ( ) ; }


Input: Try to get the Java Date / Time formatting associated with the C standard provided . [CODESPLIT] protected boolean translateCommand ( StringBuilder buf , String pattern , int index , boolean oldInside ) { char firstChar = pattern . charAt ( index ) ; boolean newInside = oldInside ; //O and E are modifiers, they mean to present an alternative representation of the next char //we just handle the next char as if the O or E wasn't there if ( firstChar == ' ' || firstChar == ' ' ) { if ( index + 1 < pattern . length ( ) ) { newInside = translateCommand ( buf , pattern , index + 1 , oldInside ) ; } else { buf . append ( quote ( "%" + firstChar , oldInside ) ) ; } } else { String command = translate . getProperty ( String . valueOf ( firstChar ) ) ; //If we don't find a format, treat it as a literal--That's what apache does if ( command == null ) { buf . append ( quote ( "%" + firstChar , oldInside ) ) ; } else { //If we were inside quotes, close the quotes if ( oldInside ) { buf . append ( ' ' ) ; } buf . append ( command ) ; newInside = false ; } } return newInside ; }


Input: / * Max Error retries > = 0 are set in ClientConfig for S3Client < 0 will use default ( 3 ) [CODESPLIT] public void init ( Stage . Context context , String configPrefix , ProxyConfig proxyConfig , List < Stage . ConfigIssue > issues , int maxErrorRetries ) { this . maxErrorRetries = maxErrorRetries ; commonPrefix = AWSUtil . normalizePrefix ( commonPrefix , delimiter ) ; try { createConnection ( context , configPrefix , proxyConfig , issues , maxErrorRetries ) ; } catch ( StageException ex ) { LOG . debug ( Errors . S3_SPOOLDIR_20 . getMessage ( ) , ex . toString ( ) , ex ) ; issues . add ( context . createConfigIssue ( Groups . S3 . name ( ) , configPrefix + S3ConnectionBaseConfig . AWS_CONFIG_PREFIX + "awsAccessKeyId" , Errors . S3_SPOOLDIR_20 , ex . toString ( ) ) ) ; } }


Input: Transition to services [CODESPLIT] private static void upgradeV5ToV6 ( List < Config > configs , Context context ) { List < Config > dataFormatConfigs = configs . stream ( ) . filter ( c -> c . getName ( ) . startsWith ( "dataFormat" ) ) . collect ( Collectors . toList ( ) ) ; // Remove those configs configs . removeAll ( dataFormatConfigs ) ; // There is an interesting history with compression - at some point (version 2), we explicitly added it, then // we have hidden it. So this config might or might not exists, depending on the version in which the pipeline // was created. However the service is expecting it and thus we need to ensure that it's there. if ( dataFormatConfigs . stream ( ) . noneMatch ( c -> "dataFormatConfig.compression" . equals ( c . getName ( ) ) ) ) { dataFormatConfigs . add ( new Config ( "dataFormatConfig.compression" , "NONE" ) ) ; } // And finally register new service context . registerService ( DataFormatParserService . class , dataFormatConfigs ) ; }


Input: Evaluates a given field path expression against a given record and returns all field paths that satisfy the expression upon evaluation . Analagous in function to { @link FieldRegexUtil#getMatchingFieldPaths ( String Set ) } and even identical in behavior to it when the given { @code fieldExpression } does not contain any EL expressions . [CODESPLIT] public static List < String > evaluateMatchingFieldPaths ( String fieldExpression , ELEval elEval , ELVars elVars , Record record , Iterable < String > recordEscapedFieldPaths ) throws ELEvalException { if ( isFieldPathExpressionFast ( fieldExpression ) ) { // this field path expression actually does contain an EL expression, so need to evaluate against all fields return evaluateMatchingFieldPathsImpl ( fieldExpression , elEval , elVars , record ) ; } else { // else it does NOT contain one, so the field regex util (which is faster) can be used return FieldRegexUtil . getMatchingFieldPaths ( fieldExpression , recordEscapedFieldPaths ) ; } }


Input: Evaluates a field expression against a specific field in a record to see if it matches [CODESPLIT] private static boolean pathMatches ( Record record , String fieldPath , String fieldExpression , ELEval elEval , ELVars elVars ) throws ELEvalException { List < PathElement > actualPathElements = PathElement . parse ( fieldPath , true ) ; List < PathElement > matcherPathElements = PathElement . parse ( fieldExpression , false , true ) ; Iterator < PathElement > currentPathIter = actualPathElements . iterator ( ) ; Iterator < PathElement > matcherPathIter = matcherPathElements . iterator ( ) ; PathElement currentMatcher = null ; PathElement currentPath = null ; Field currentField = null ; StringBuilder currentFieldPath = new StringBuilder ( ) ; while ( matcherPathIter . hasNext ( ) ) { currentMatcher = matcherPathIter . next ( ) ; switch ( currentMatcher . getType ( ) ) { case MAP : if ( ! currentPathIter . hasNext ( ) ) { // we are expecting to match a MAP, but there are no more elements in the path return false ; } currentPath = currentPathIter . next ( ) ; // see if the name matches the pattern String childName = currentPath . getName ( ) ; String patternName = currentMatcher . getName ( ) ; if ( FieldRegexUtil . hasWildCards ( patternName ) ) { patternName = FieldRegexUtil . transformFieldPathRegex ( patternName ) ; } if ( childName . matches ( patternName ) ) { currentField = currentField . getValueAsMap ( ) . get ( childName ) ; currentFieldPath . append ( "/" ) ; currentFieldPath . append ( childName ) ; } else { return false ; } break ; case LIST : // see if the index matches the pattern if ( ! currentPathIter . hasNext ( ) ) { // we are expecting to match a LIST, but there are no more elements in the path return false ; } currentPath = currentPathIter . next ( ) ; int childIndex = currentPath . getIndex ( ) ; final int matchInd = currentMatcher . getIndex ( ) ; if ( matchInd == PathElement . WILDCARD_INDEX_ANY_LENGTH || ( matchInd == PathElement . WILDCARD_INDEX_SINGLE_CHAR && childIndex < 10 ) || matchInd == childIndex ) { currentField = currentField . getValueAsList ( ) . get ( childIndex ) ; currentFieldPath . append ( "[" ) ; currentFieldPath . append ( childIndex ) ; currentFieldPath . append ( "]" ) ; } else { return false ; } break ; case ROOT : if ( ! currentPathIter . hasNext ( ) ) { // we are expecting to match the ROOT, but there are no more elements in the path return false ; } currentPath = currentPathIter . next ( ) ; if ( currentPath . getType ( ) != PathElement . Type . ROOT ) { // we are expecting to match the ROOT, the root element wasn't for some reason return false ; } currentField = record . get ( ) ; break ; case FIELD_EXPRESSION : // see if the current field matches the given expression FieldEL . setFieldInContext ( elVars , currentFieldPath . toString ( ) , currentPath . getName ( ) , currentField ) ; String expression = currentMatcher . getName ( ) ; final boolean result = elEval . eval ( elVars , expression , Boolean . class ) ; if ( LOG . isDebugEnabled ( ) ) { LOG . debug ( "Result of evaluating expression {} on field {} with path {} was {}" , expression , currentField , currentFieldPath , result ) ; } if ( ! result ) { return false ; } break ; } } return ! currentPathIter . hasNext ( ) ; }


Input: This method must be used after completing the write to output stream which was obtained by calling the { @link #getOutputStream () } method . [CODESPLIT] public void release ( ) { CounterLock lock ; synchronized ( DataStore . class ) { lock = FILE_LOCKS . get ( file ) ; if ( lock == null ) { LOG . error ( "Trying to release unlocked file {}" , file ) ; return ; } lock . dec ( ) ; if ( lock . counter == 0 ) { FILE_LOCKS . remove ( file ) ; } } LOG . trace ( "Releasing the lock {} for '{}'" , lock , file ) ; lock . unlock ( ) ; LOG . trace ( "Released the lock {} for '{}'" , lock , file ) ; }


Input: Returns an input stream for the requested file . [CODESPLIT] public InputStream getInputStream ( ) throws IOException { acquireLock ( ) ; try { isClosed = false ; forWrite = false ; LOG . trace ( "Starts read '{}'" , file ) ; verifyAndRecover ( ) ; InputStream is = new ProxyInputStream ( new FileInputStream ( file . toFile ( ) ) ) { @ Override public void close ( ) throws IOException { if ( isClosed ) { return ; } try { super . close ( ) ; } finally { release ( ) ; isClosed = true ; stream = null ; } LOG . trace ( "Finishes read '{}'" , file ) ; } } ; stream = is ; return is ; } catch ( Exception ex ) { release ( ) ; throw ex ; } }


Input: Returns an output stream for the requested file . [CODESPLIT] public OutputStream getOutputStream ( ) throws IOException { acquireLock ( ) ; try { isClosed = false ; forWrite = true ; LOG . trace ( "Starts write '{}'" , file ) ; verifyAndRecover ( ) ; if ( Files . exists ( file ) ) { Files . move ( file , fileOld ) ; LOG . trace ( "Starting write, move '{}' to '{}'" , file , fileOld ) ; } OutputStream os = new ProxyOutputStream ( new FileOutputStream ( fileTmp . toFile ( ) ) ) { @ Override public void close ( ) throws IOException { if ( isClosed ) { return ; } try { super . close ( ) ; } finally { isClosed = true ; stream = null ; } LOG . trace ( "Finishes write '{}'" , file ) ; } } ; stream = os ; return os ; } catch ( Exception ex ) { release ( ) ; throw ex ; } }


Input: This method must be used to commit contents written to the output stream which is obtained by calling the { @link #getOutputStream () } method . This method closes the argument output stream . [CODESPLIT] public void commit ( OutputStream out ) throws IOException { // close the stream in order to flush the contents into the disk Utils . checkNotNull ( out , "Argument output stream cannot be null" ) ; Utils . checkState ( stream == out , "The argument output stream must be the same as the output stream obtained " + "from this data store instance" ) ; out . close ( ) ; Files . move ( fileTmp , fileNew ) ; LOG . trace ( "Committing write, move '{}' to '{}'" , fileTmp , fileNew ) ; Files . move ( fileNew , file ) ; LOG . trace ( "Committing write, move '{}' to '{}'" , fileNew , file ) ; if ( Files . exists ( fileOld ) ) { Files . delete ( fileOld ) ; LOG . trace ( "Committing write, deleting '{}'" , fileOld ) ; } LOG . trace ( "Committed" ) ; }


Input: Check if the DataStore exists and contains data . This method will check for the presence of the set of files that can be used to read data from the store . [CODESPLIT] public boolean exists ( ) throws IOException { acquireLock ( ) ; try { verifyAndRecover ( ) ; return Files . exists ( file ) && Files . size ( file ) > 0 ; } finally { release ( ) ; } }


Input: Get an operation code from record . First look for sdc . operation . code from record header . If not set look for __$operation in record . It is a specific field that MS SQL CDC origin set . [CODESPLIT] @ Override @ VisibleForTesting int getOperationFromRecord ( Record record , int defaultOpCode , UnsupportedOperationAction unsupportedAction , List < OnRecordErrorException > errorRecords ) { int opCode = - 1 ; // -1 is invalid and not used in OperationType. String op = null ; try { // Try sdc.operation.type first op = record . getHeader ( ) . getAttribute ( OperationType . SDC_OPERATION_TYPE ) ; // If not set, look for "__$operation" in record. if ( StringUtils . isBlank ( op ) ) { if ( record . has ( MSOperationCode . getOpField ( ) ) ) { int intOp = record . get ( MSOperationCode . getOpField ( ) ) . getValueAsInteger ( ) ; // Convert the MS specific operation code to SDC standard operation code opCode = MSOperationCode . convertToJDBCCode ( intOp ) ; } } else { opCode = JDBCOperationType . convertToIntCode ( op ) ; } if ( opCode == - 1 ) { // Both MS code and sdc code are not set. Use default. opCode = defaultOpCode ; } } catch ( NumberFormatException | UnsupportedOperationException ex ) { LOG . debug ( "Operation obtained from record is not supported: {}. Handle by UnsupportedOpertaionAction {}. {}" , ex . getMessage ( ) , unsupportedAction . getLabel ( ) , ex ) ; switch ( unsupportedAction ) { case DISCARD : LOG . debug ( "Discarding record with unsupported operation {}" , op ) ; break ; case SEND_TO_ERROR : LOG . debug ( "Sending record to error due to unsupported operation {}" , op ) ; errorRecords . add ( new OnRecordErrorException ( record , JdbcErrors . JDBC_70 , op ) ) ; break ; case USE_DEFAULT : opCode = defaultOpCode ; break ; default : //unknown action LOG . debug ( "Sending record to error due to unknown operation: {}" , op ) ; } } return opCode ; }


Input: Parse String representation of permissions into HDFS FsPermission class . [CODESPLIT] public static FsPermission parseFsPermission ( String permissions ) throws IllegalArgumentException { try { // Octal or symbolic representation return new FsPermission ( permissions ) ; } catch ( IllegalArgumentException e ) { // FsPermission.valueOf will work with unix style permissions which is 10 characters // where the first character says the type of file if ( permissions . length ( ) == 9 ) { // This means it is a posix standard without the first character for file type // We will simply set it to '-' suggesting regular file permissions = "-" + permissions ; } // Try to parse unix style format. return FsPermission . valueOf ( permissions ) ; } }


Input: Validate that all required libraries are available and loaded . [CODESPLIT] @ VisibleForTesting void validateRequiredStageLibraries ( ) { // Name of all installed libraries Set < String > installedLibraries = stageLibraries . stream ( ) . map ( StageLibraryDefinition :: getName ) . collect ( Collectors . toSet ( ) ) ; // Required libraries Set < String > requiredLibraries = new HashSet <> ( ) ; String config = configuration . get ( SdcConfiguration . REQUIRED_STAGELIBS , DEFAULT_REQUIRED_STAGELIBS ) ; for ( String stageLib : config . split ( "," ) ) { if ( ! stageLib . isEmpty ( ) ) { requiredLibraries . add ( stageLib ) ; } } Set < String > missingLibraries = Sets . difference ( requiredLibraries , installedLibraries ) ; if ( ! missingLibraries . isEmpty ( ) ) { throw new RuntimeException ( Utils . format ( "Some required stage libraries are missing: {}" , StringUtils . join ( missingLibraries , ", " ) ) ) ; } }


Input: Validate service dependencies . [CODESPLIT] private void validateAllServicesAvailable ( ) { // Firstly validate that all stages have satisfied service dependencies List < String > missingServices = new LinkedList <> ( ) ; for ( StageDefinition stage : stageList ) { for ( ServiceDependencyDefinition service : stage . getServices ( ) ) { if ( ! serviceMap . containsKey ( service . getService ( ) ) ) { missingServices . add ( Utils . format ( "Stage {} is missing service {}" , stage . getName ( ) , service . getService ( ) . getName ( ) ) ) ; } } } if ( ! missingServices . isEmpty ( ) ) { throw new RuntimeException ( "Missing services: " + StringUtils . join ( missingServices , ", " ) ) ; } // Secondly ensure that all loaded services are compatible with what is supported by our runtime engine List < String > unsupportedServices = new LinkedList <> ( ) ; for ( ServiceDefinition serviceDefinition : serviceList ) { if ( ! ServiceRuntime . supports ( serviceDefinition . getProvides ( ) ) ) { unsupportedServices . add ( serviceDefinition . getProvides ( ) . toString ( ) ) ; } } if ( ! unsupportedServices . isEmpty ( ) ) { throw new RuntimeException ( "Unsupported services: " + StringUtils . join ( unsupportedServices , ", " ) ) ; } }


Input: Creates a simple Aggregator . [CODESPLIT] @ SuppressWarnings ( "unchecked" ) public < A extends SimpleAggregator > A createSimple ( String name , Class < ? extends Aggregator > klass ) { Utils . checkState ( ! started , "Already started" ) ; try { A aggregator = ( A ) CONSTRUCTORS . get ( klass ) . newInstance ( name ) ; dataProvider . addAggregator ( aggregator ) ; aggregator . setDataProvider ( dataProvider ) ; return aggregator ; } catch ( Exception ex ) { throw new RuntimeException ( ex ) ; } }


Input: Returns the unit type of an aggregator value . Typically Long or Double . [CODESPLIT] < A extends SimpleAggregator , T > Class < ? extends Number > getAggregatorUnit ( Class < A > klass ) { try { A aggregator = ( A ) CONSTRUCTORS . get ( klass ) . newInstance ( "forAggregatorTypeDiscoveryOnly" ) ; return aggregator . getValueType ( ) ; } catch ( Exception ex ) { throw new RuntimeException ( ex ) ; } }


Input: Creates an AggregatorData . [CODESPLIT] < A extends SimpleAggregator , T > AggregatorData < A , T > createAggregatorData ( Class < A > klass , String name , long timeWindowMillis ) { try { A aggregator = ( A ) CONSTRUCTORS . get ( klass ) . newInstance ( name ) ; return aggregator . createAggregatorData ( timeWindowMillis ) ; } catch ( Exception ex ) { throw new RuntimeException ( ex ) ; } }


Input: Creates a group - by Agregator . [CODESPLIT] @ SuppressWarnings ( "unchecked" ) public < A extends SimpleAggregator , N extends Number > GroupByAggregator < A , N > createGroupBy ( String name , Class < ? extends Aggregator > aKlass ) { Utils . checkState ( ! started , "Already started" ) ; GroupByAggregator < A , N > aggregator = new GroupByAggregator ( name , aKlass , this ) ; dataProvider . addAggregator ( aggregator ) ; aggregator . setDataProvider ( dataProvider ) ; return aggregator ; }


Input: Starts the Aggregators instance . [CODESPLIT] public void start ( long newDataWindowEndTimeMillis ) { Utils . checkState ( ! started , "Already started" ) ; Utils . checkState ( ! stopped , "Already stopped" ) ; dataProvider . start ( newDataWindowEndTimeMillis ) ; started = true ; }


Input: Stops the Aggregators instance . [CODESPLIT] public Map < Aggregator , AggregatorData > stop ( ) { Utils . checkState ( started , "Already started" ) ; Utils . checkState ( ! stopped , "Already stopped" ) ; Map < Aggregator , AggregatorData > aggregatorDataMap = dataProvider . stop ( ) ; stopped = true ; return aggregatorDataMap ; }


Input: Atomically rolls the DataWindow of all aggregators associated with the Aggregators instance . [CODESPLIT] public Map < Aggregator , AggregatorData > roll ( long newDataWindowEndTimeMillis ) { Utils . checkState ( started , "Not started" ) ; Utils . checkState ( ! stopped , "Already stopped" ) ; return dataProvider . roll ( newDataWindowEndTimeMillis ) ; }


Input: Evaluate precision or scale in context of record and given field path . [CODESPLIT] private static int resolveScaleOrPrecisionExpression ( String type , Field field , String attributeName , String fieldPath ) throws JdbcStageCheckedException { String stringValue = field . getAttribute ( attributeName ) ; try { return Integer . parseInt ( stringValue ) ; } catch ( NumberFormatException e ) { throw new JdbcStageCheckedException ( JdbcErrors . JDBC_304 , type , fieldPath , attributeName , stringValue , e ) ; } }


Input: Returns an instance of LoginManager and increases its reference count . [CODESPLIT] public static final LoginManager acquireLoginManager ( LoginType loginType , Map < String , ? > configs ) throws IOException , LoginException { synchronized ( LoginManager . class ) { LoginManager loginManager = CACHED_INSTANCES . get ( loginType ) ; if ( loginManager == null ) { loginManager = new LoginManager ( loginType , configs ) ; CACHED_INSTANCES . put ( loginType , loginManager ) ; } return loginManager . acquire ( ) ; } }


Input: Decrease the reference count for this instance and release resources if it reaches 0 . [CODESPLIT] public void release ( ) { synchronized ( LoginManager . class ) { if ( refCount == 0 ) throw new IllegalStateException ( "release called on LoginManager with refCount == 0" ) ; else if ( refCount == 1 ) { CACHED_INSTANCES . remove ( loginType ) ; login . shutdown ( ) ; } -- refCount ; } }


Input: / * Should only be used in tests . [CODESPLIT] public static void closeAll ( ) { synchronized ( LoginManager . class ) { for ( LoginType loginType : new ArrayList <> ( CACHED_INSTANCES . keySet ( ) ) ) { LoginManager loginManager = CACHED_INSTANCES . remove ( loginType ) ; loginManager . login . shutdown ( ) ; } } }


Input: Convert a limited file glob into a simple regex . [CODESPLIT] private static String globToRegex ( String glob ) { if ( glob . charAt ( 0 ) == ' ' || glob . contains ( "/" ) || glob . contains ( "~" ) ) { throw new IllegalArgumentException ( "Invalid character in file glob" ) ; } // treat dot as a literal. glob = glob . replace ( "." , "\\." ) ; glob = glob . replace ( "*" , ".+" ) ; glob = glob . replace ( "?" , ".{1}+" ) ; return glob ; }


Input: Decodes one or more { @link NetflowV5Message } from a packet . For this method the data ( ByteBuf ) is assumed to be complete ( i . e . it will contain all the data ) which is the case for UDP . [CODESPLIT] public void decodeStandaloneBuffer ( ByteBuf buf , List < BaseNetflowMessage > resultMessages , InetSocketAddress sender , InetSocketAddress recipient ) throws OnRecordErrorException { final List < Object > results = new LinkedList <> ( ) ; try { decode ( null , buf , results , sender , recipient , true ) ; for ( Object result : results ) { if ( result == null ) { LOG . warn ( "null result found from decoding standalone Netflow buffer; skipping" ) ; continue ; } if ( result instanceof BaseNetflowMessage ) { resultMessages . add ( ( BaseNetflowMessage ) result ) ; } else { throw new IllegalStateException ( String . format ( "Found unexpected object type in results: %s" , result . getClass ( ) . getName ( ) ) ) ; } } } finally { resetStateVariables ( ) ; } }


Input: Lists objects from AmazonS3 in lexicographical order [CODESPLIT] static List < S3ObjectSummary > listObjectsLexicographically ( AmazonS3 s3Client , S3ConfigBean s3ConfigBean , AntPathMatcher pathMatcher , S3Offset s3Offset , int fetchSize ) { // Incrementally scan objects after the marker (s3Offset). List < S3ObjectSummary > list = new ArrayList <> ( fetchSize ) ; ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) ; listObjectsRequest . setBucketName ( s3ConfigBean . s3Config . bucket ) ; listObjectsRequest . setPrefix ( s3ConfigBean . s3Config . commonPrefix ) ; listObjectsRequest . setMaxKeys ( BATCH_SIZE ) ; if ( s3Offset . getKey ( ) != null ) { listObjectsRequest . setMarker ( s3Offset . getKey ( ) ) ; } ObjectListing objectListing = s3Client . listObjects ( listObjectsRequest ) ; while ( true ) { for ( S3ObjectSummary s : objectListing . getObjectSummaries ( ) ) { String fullPrefix = s . getKey ( ) ; String remainingPrefix = fullPrefix . substring ( s3ConfigBean . s3Config . commonPrefix . length ( ) , fullPrefix . length ( ) ) ; if ( ! remainingPrefix . isEmpty ( ) ) { if ( pathMatcher . match ( s3ConfigBean . s3FileConfig . prefixPattern , remainingPrefix ) ) { list . add ( s ) ; } // We've got enough objects. if ( list . size ( ) == fetchSize ) { return list ; } } } // Listing is complete. No more objects to be listed. if ( ! objectListing . isTruncated ( ) ) { break ; } objectListing = s3Client . listNextBatchOfObjects ( objectListing ) ; } return list ; }


Input: Lists objects from AmazonS3 in chronological order [ lexicographical order if 2 files have same timestamp ] which are later than or equal to the timestamp of the previous offset object [CODESPLIT] static List < S3ObjectSummary > listObjectsChronologically ( AmazonS3 s3Client , S3ConfigBean s3ConfigBean , AntPathMatcher pathMatcher , S3Offset s3Offset , int fetchSize ) { //Algorithm: // - Full scan all objects that match the file name pattern and which are later than the file in the offset // - Select the oldest "fetchSize" number of files and return them. TreeSet < S3ObjectSummary > treeSet = new TreeSet <> ( ( o1 , o2 ) -> { int result = o1 . getLastModified ( ) . compareTo ( o2 . getLastModified ( ) ) ; if ( result != 0 ) { //same modified time. Use name to sort return result ; } return o1 . getKey ( ) . compareTo ( o2 . getKey ( ) ) ; } ) ; S3Objects s3ObjectSummaries = S3Objects . withPrefix ( s3Client , s3ConfigBean . s3Config . bucket , s3ConfigBean . s3Config . commonPrefix ) ; // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call for ( S3ObjectSummary s : s3ObjectSummaries ) { String fullPrefix = s . getKey ( ) ; String remainingPrefix = fullPrefix . substring ( s3ConfigBean . s3Config . commonPrefix . length ( ) , fullPrefix . length ( ) ) ; if ( ! remainingPrefix . isEmpty ( ) ) { // remainingPrefix can be empty. // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject", // then the first objects returned here are: // myFolder/mySubFolder // myFolder/mySubFolder/myObject // // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it // has no data if ( pathMatcher . match ( s3ConfigBean . s3FileConfig . prefixPattern , remainingPrefix ) && isEligible ( s , s3Offset ) ) { treeSet . add ( s ) ; } if ( treeSet . size ( ) > fetchSize ) { treeSet . pollLast ( ) ; } } } return new ArrayList <> ( treeSet ) ; }


Input: / * ------------------------------------------------------------ [CODESPLIT] @ Override protected String [ ] loadRoleInfo ( UserPrincipal user ) { UserIdentity id = _userStore . getUserIdentity ( user . getName ( ) ) ; if ( id == null ) return null ; Set < RolePrincipal > roles = id . getSubject ( ) . getPrincipals ( RolePrincipal . class ) ; if ( roles == null ) return null ; List < String > list = roles . stream ( ) . map ( RolePrincipal :: getName ) . collect ( Collectors . toList ( ) ) ; return list . toArray ( new String [ roles . size ( ) ] ) ; }


Input: / * ------------------------------------------------------------ [CODESPLIT] @ Override protected UserPrincipal loadUserInfo ( String userName ) { UserIdentity id = _userStore . getUserIdentity ( userName ) ; if ( id != null ) { return ( UserPrincipal ) id . getUserPrincipal ( ) ; } return null ; }


Input: Since stages are allowed to produce events during destroy () phase we handle the destroy event as simplified runBatch . We go over all the StagePipes and if it s on data path we destroy it immediately if it s on event path we run it one more time . Since the stages are sorted we know that destroyed stage will never be needed again . Non stage pipes are always processed to generate required structures in PipeBatch . [CODESPLIT] @ Override public void destroy ( SourcePipe originPipe , List < PipeRunner > pipeRunners , BadRecordsHandler badRecordsHandler , StatsAggregationHandler statsAggregationHandler ) throws StageException , PipelineRuntimeException { // We're no longer running running = false ; // There are two ways a runner can be in use - used by real runner (e.g. when origin produced data) or when it's // processing "empty" batch when the runner was idle for too long. The first case is guarded by the framework - this // method won't be called until the execution successfully finished. However the second way with idle drivers is run // from a separate thread and hence we have to ensure here that it's "done" before moving on with the destroy phase. try { destroyLock . lock ( ) ; // Firstly destroy the runner, to make sure that any potential run away thread from origin will be denied // further processing. if ( runnerPool != null ) { runnerPool . destroy ( ) ; } int batchSize = configuration . get ( Constants . MAX_BATCH_SIZE_KEY , Constants . MAX_BATCH_SIZE_DEFAULT ) ; long lastBatchTime = offsetTracker . getLastBatchTime ( ) ; long start = System . currentTimeMillis ( ) ; FullPipeBatch pipeBatch ; // Destroy origin pipe pipeBatch = new FullPipeBatch ( null , null , batchSize , false ) ; try { LOG . trace ( "Destroying origin pipe" ) ; pipeBatch . skipStage ( originPipe ) ; originPipe . destroy ( pipeBatch ) ; } catch ( RuntimeException e ) { LOG . warn ( "Exception throw while destroying pipe" , e ) ; } // Now destroy the pipe runners // // We're destroying them in reverser order to make sure that the last runner to destroy is the one with id '0' // that holds reference to all class loaders. Runners with id >0 do not own their class loaders and hence needs to // be destroyed before the runner with id '0'. for ( PipeRunner pipeRunner : Lists . reverse ( pipeRunners ) ) { final FullPipeBatch finalPipeBatch = pipeBatch ; pipeRunner . executeBatch ( null , null , start , pipe -> { // Set the last batch time in the stage context of each pipe ( ( StageContext ) pipe . getStage ( ) . getContext ( ) ) . setLastBatchTime ( lastBatchTime ) ; String instanceName = pipe . getStage ( ) . getConfiguration ( ) . getInstanceName ( ) ; if ( pipe instanceof StagePipe ) { // Stage pipes are processed only if they are in event path if ( pipe . getStage ( ) . getConfiguration ( ) . isInEventPath ( ) ) { LOG . trace ( "Stage pipe {} is in event path, running last process" , instanceName ) ; pipe . process ( finalPipeBatch ) ; } else { LOG . trace ( "Stage pipe {} is in data path, skipping it's processing." , instanceName ) ; finalPipeBatch . skipStage ( pipe ) ; } } else { // Non stage pipes are executed always LOG . trace ( "Non stage pipe {}, running last process" , instanceName ) ; pipe . process ( finalPipeBatch ) ; } // And finally destroy the pipe try { LOG . trace ( "Running destroy for {}" , instanceName ) ; pipe . destroy ( finalPipeBatch ) ; } catch ( RuntimeException e ) { LOG . warn ( "Exception throw while destroying pipe" , e ) ; } } ) ; badRecordsHandler . handle ( null , null , pipeBatch . getErrorSink ( ) , pipeBatch . getSourceResponseSink ( ) ) ; // Next iteration should have new and empty PipeBatch pipeBatch = new FullPipeBatch ( null , null , batchSize , false ) ; pipeBatch . skipStage ( originPipe ) ; } if ( isStatsAggregationEnabled ( ) ) { List < Record > stats = new ArrayList <> ( ) ; statsAggregatorRequests . drainTo ( stats ) ; Object timeSeriesString = pipelineConfigBean . constants . get ( MetricsEventRunnable . TIME_SERIES_ANALYSIS ) ; boolean timeSeriesAnalysis = ( timeSeriesString != null ) ? ( Boolean ) timeSeriesString : true ; String metricRegistryStr ; try { metricRegistryStr = ObjectMapperFactory . get ( ) . writer ( ) . writeValueAsString ( metrics ) ; } catch ( Exception e ) { throw new RuntimeException ( Utils . format ( "Error converting metric json to string: {}" , e ) , e ) ; } LOG . info ( "Queueing last batch of record to be sent to stats aggregator" ) ; stats . add ( AggregatorUtil . createMetricJsonRecord ( runtimeInfo . getId ( ) , runtimeInfo . getMasterSDCId ( ) , pipelineConfiguration . getMetadata ( ) , false , timeSeriesAnalysis , true , metricRegistryStr ) ) ; statsAggregationHandler . handle ( null , null , stats ) ; } } finally { destroyLock . unlock ( ) ; } }


Input: Stops execution of the pipeline after the current batch completes [CODESPLIT] public void stop ( ) throws PipelineException { this . stop = true ; if ( batchesToCapture > 0 ) { cancelSnapshot ( this . snapshotName ) ; snapshotStore . deleteSnapshot ( pipelineName , revision , snapshotName ) ; } }


Input: This method should be called periodically from a scheduler if the pipeline should not allow runners to be idle for more then idleTime . [CODESPLIT] public int produceEmptyBatchesForIdleRunners ( long idleTime ) throws PipelineException , StageException { LOG . debug ( "Checking if any active runner is idle" ) ; // The empty batch is suppose to be fast - almost as a zero time. It could however happened that from some reason it // will take a long time (possibly more then idleTime). To avoid infinite loops, this method will only processes up // to total number of runners before returning. int counter = 0 ; try { destroyLock . lock ( ) ; while ( running && counter < pipes . size ( ) ) { counter ++ ; PipeRunner runner = null ; try { runner = runnerPool . getIdleRunner ( idleTime ) ; // No more idle runners, simply stop the idle execution now if ( runner == null ) { return counter ; } LOG . debug ( "Generating empty batch for runner: {}" , runner . getRunnerId ( ) ) ; pipeContext . getRuntimeStats ( ) . incIdleBatchCount ( ) ; // Pipe batch to keep the batch info FullPipeBatch pipeBatch = new FullPipeBatch ( null , null , 0 , false ) ; pipeBatch . setIdleBatch ( true ) ; // We're explicitly skipping origin because this is framework generated, empty batch pipeBatch . skipStage ( originPipe ) ; executeRunner ( runner , System . currentTimeMillis ( ) , pipeBatch , null , null , new HashMap <> ( ) , new HashMap <> ( ) ) ; } finally { if ( runner != null ) { runnerPool . returnRunner ( runner ) ; } } } } finally { destroyLock . unlock ( ) ; } return counter ; }


Input: Create special batch by salvaging memory structures when pipelines gets into un - recoverable error . [CODESPLIT] private void createFailureBatch ( FullPipeBatch pipeBatch ) { if ( ! pipelineConfigBean . shouldCreateFailureSnapshot ) { return ; } try { for ( SnapshotInfo info : snapshotStore . getSummaryForPipeline ( pipelineName , revision ) ) { // Allow only one failure snapshot to be present on a pipeline if ( info . isFailureSnapshot ( ) ) { LOG . trace ( "Skipping creation of failure snapshot as {} already exists." , info . getId ( ) ) ; return ; } } String snapshotName = "Failure_" + UUID . randomUUID ( ) . toString ( ) ; String snapshotLabel = "Failure at " + LocalDateTime . now ( ) . toString ( ) ; snapshotStore . create ( "" , pipelineName , revision , snapshotName , snapshotLabel , true ) ; snapshotStore . save ( pipelineName , revision , snapshotName , - 1 , ImmutableList . of ( pipeBatch . createFailureSnapshot ( ) ) ) ; } catch ( PipelineException ex ) { LOG . error ( "Can't serialize failure snapshot" , ex ) ; } }


Input: Convert a Record into a fully - bound statement . [CODESPLIT] @ SuppressWarnings ( "unchecked" ) private BoundStatement recordToBoundStatement ( Record record ) throws StageException { ImmutableList . Builder < Object > values = new ImmutableList . Builder <> ( ) ; SortedSet < String > columnsPresent = Sets . newTreeSet ( columnMappings . keySet ( ) ) ; for ( Map . Entry < String , String > mapping : columnMappings . entrySet ( ) ) { String columnName = mapping . getKey ( ) ; String fieldPath = mapping . getValue ( ) ; // If we're missing fields, skip them. // If a field is present, but null, also remove it from columnsPresent since we can't write nulls. if ( ! record . has ( fieldPath ) || record . get ( fieldPath ) . getValue ( ) == null ) { columnsPresent . remove ( columnName ) ; continue ; } final Object value = record . get ( fieldPath ) . getValue ( ) ; // Special cases for handling SDC Lists and Maps, // basically unpacking them into raw types. if ( value instanceof List ) { List < Object > unpackedList = new ArrayList <> ( ) ; for ( Field item : ( List < Field > ) value ) { unpackedList . add ( item . getValue ( ) ) ; } values . add ( unpackedList ) ; } else if ( value instanceof Map ) { Map < Object , Object > unpackedMap = new HashMap <> ( ) ; for ( Map . Entry < String , Field > entry : ( ( Map < String , Field > ) value ) . entrySet ( ) ) { unpackedMap . put ( entry . getKey ( ) , entry . getValue ( ) . getValue ( ) ) ; } values . add ( unpackedMap ) ; } else { values . add ( value ) ; } } PreparedStatement stmt = statementCache . getUnchecked ( columnsPresent ) ; // .toArray required to pass in a list to a varargs method. Object [ ] valuesArray = values . build ( ) . toArray ( ) ; BoundStatement boundStmt = null ; try { boundStmt = stmt . bind ( valuesArray ) ; } catch ( CodecNotFoundException | InvalidTypeException | NullPointerException e ) { // NPE can occur if one of the values is a collection type with a null value inside it. Thus, it's a record // error. Note that this runs the risk of mistakenly treating a bug as a record error. // CodecNotFound is caused when there is no type conversion definition available from the provided type // to the target type. errorRecordHandler . onError ( new OnRecordErrorException ( record , Errors . CASSANDRA_06 , record . getHeader ( ) . getSourceId ( ) , e . toString ( ) , e ) ) ; } return boundStmt ; }


Input: { [CODESPLIT] @ Override protected List < ConfigIssue > init ( ) { // Validate configuration values and open any required resources. List < ConfigIssue > issues = super . init ( ) ; Target . Context context = getContext ( ) ; Optional . ofNullable ( conf . init ( context , CONF_PREFIX ) ) . ifPresent ( issues :: addAll ) ; errorRecordHandler = new DefaultErrorRecordHandler ( context ) ; sObjectNameVars = getContext ( ) . createELVars ( ) ; sObjectNameEval = context . createELEval ( SOBJECT_NAME ) ; ELUtils . validateExpression ( conf . sObjectNameTemplate , context , Groups . FORCE . getLabel ( ) , SOBJECT_NAME , Errors . FORCE_12 , issues ) ; externalIdFieldVars = getContext ( ) . createELVars ( ) ; externalIdFieldEval = context . createELEval ( EXTERNAL_ID_NAME ) ; ELUtils . validateExpression ( conf . externalIdField , context , Groups . FORCE . getLabel ( ) , EXTERNAL_ID_NAME , Errors . FORCE_24 , issues ) ; if ( issues . isEmpty ( ) ) { fieldMappings = new TreeMap <> ( ) ; for ( ForceFieldMapping mapping : conf . fieldMapping ) { // SDC-7446 Allow colon as well as period as field separator String salesforceField = conf . useBulkAPI ? mapping . salesforceField . replace ( ' ' , ' ' ) : mapping . salesforceField ; fieldMappings . put ( salesforceField , mapping . sdcField ) ; } try { ConnectorConfig partnerConfig = ForceUtils . getPartnerConfig ( conf , new ForceSessionRenewer ( ) ) ; partnerConnection = Connector . newConnection ( partnerConfig ) ; if ( conf . mutualAuth . useMutualAuth ) { ForceUtils . setupMutualAuth ( partnerConfig , conf . mutualAuth ) ; } bulkConnection = ForceUtils . getBulkConnection ( partnerConfig , conf ) ; LOG . info ( "Successfully authenticated as {}" , conf . username ) ; } catch ( ConnectionException | AsyncApiException | StageException | URISyntaxException ce ) { LOG . error ( "Can't connect to SalesForce" , ce ) ; issues . add ( getContext ( ) . createConfigIssue ( Groups . FORCE . name ( ) , "connectorConfig" , Errors . FORCE_00 , ForceUtils . getExceptionCode ( ce ) + ", " + ForceUtils . getExceptionMessage ( ce ) ) ) ; } if ( conf . useBulkAPI ) { writer = new ForceBulkWriter ( fieldMappings , bulkConnection , getContext ( ) ) ; } else { writer = new ForceSoapWriter ( fieldMappings , partnerConnection ) ; } } // If issues is not empty, the UI will inform the user of each configuration issue in the list. return issues ; }


Input: { [CODESPLIT] @ Override public void write ( Batch batch ) throws StageException { Multimap < String , Record > partitions = ELUtils . partitionBatchByExpression ( sObjectNameEval , sObjectNameVars , conf . sObjectNameTemplate , batch ) ; Set < String > sObjectNames = partitions . keySet ( ) ; for ( String sObjectName : sObjectNames ) { List < OnRecordErrorException > errors = writer . writeBatch ( sObjectName , partitions . get ( sObjectName ) , this ) ; for ( OnRecordErrorException error : errors ) { errorRecordHandler . onError ( error ) ; } } }


Input: Set multiple configs at once . [CODESPLIT] public void set ( Map < String , String > newConfiguration ) { for ( Map . Entry < String , String > entry : newConfiguration . entrySet ( ) ) { if ( entry . getValue ( ) == null ) { this . unset ( entry . getKey ( ) ) ; } else { this . set ( entry . getKey ( ) , entry . getValue ( ) ) ; } } }


Input: { [CODESPLIT] @ Override protected void process ( Record record , SingleLaneProcessor . SingleLaneBatchMaker batchMaker ) throws StageException { RecordEL . setRecordInContext ( tableNameVars , record ) ; String tableName = tableNameEval . eval ( tableNameVars , conf . kuduTableTemplate , String . class ) ; if ( ! conf . caseSensitive ) { tableName = tableName . toLowerCase ( ) ; } LOG . trace ( "Processing record:{}  TableName={}" , record . toString ( ) , tableName ) ; try { try { KuduLookupKey key = generateLookupKey ( record , tableName ) ; List < Map < String , Field > > values = cache . get ( key ) ; if ( values . isEmpty ( ) ) { // No record found if ( conf . missingLookupBehavior == MissingValuesBehavior . SEND_TO_ERROR ) { errorRecordHandler . onError ( new OnRecordErrorException ( record , Errors . KUDU_31 ) ) ; } else { // Configured to 'Send to next stage' and 'pass as it is' batchMaker . addRecord ( record ) ; } } else { switch ( conf . multipleValuesBehavior ) { case FIRST_ONLY : setFieldsInRecord ( record , values . get ( 0 ) ) ; batchMaker . addRecord ( record ) ; break ; case SPLIT_INTO_MULTIPLE_RECORDS : for ( Map < String , Field > lookupItem : values ) { Record newRecord = getContext ( ) . cloneRecord ( record ) ; setFieldsInRecord ( newRecord , lookupItem ) ; batchMaker . addRecord ( newRecord ) ; } break ; default : throw new IllegalStateException ( "Unknown multiple value behavior: " + conf . multipleValuesBehavior ) ; } } } catch ( ExecutionException e ) { Throwables . propagateIfPossible ( e . getCause ( ) , StageException . class ) ; Throwables . propagateIfPossible ( e . getCause ( ) , OnRecordErrorException . class ) ; throw new IllegalStateException ( e ) ; // The cache loader shouldn't throw anything that isn't a StageException. } } catch ( OnRecordErrorException error ) { // NOSONAR errorRecordHandler . onError ( new OnRecordErrorException ( record , error . getErrorCode ( ) , error . getParams ( ) ) ) ; } }


Input: Create a map of keyColumn - value to lookup in cache . [CODESPLIT] private KuduLookupKey generateLookupKey ( final Record record , final String tableName ) throws OnRecordErrorException { Map < String , Field > keyList = new HashMap <> ( ) ; for ( Map . Entry < String , String > key : columnToField . entrySet ( ) ) { String fieldName = key . getValue ( ) ; if ( ! record . has ( fieldName ) ) { throw new OnRecordErrorException ( record , Errors . KUDU_32 , fieldName ) ; } keyList . put ( key . getKey ( ) , record . get ( fieldName ) ) ; } return new KuduLookupKey ( tableName , keyList ) ; }


Input: no trespassing ... [CODESPLIT] private Properties getKafkaProperties ( Stage . Context context ) { Properties props = new Properties ( ) ; props . putAll ( conf . kafkaOptions ) ; props . setProperty ( "bootstrap.servers" , conf . brokerURI ) ; props . setProperty ( "group.id" , conf . consumerGroup ) ; props . setProperty ( "max.poll.records" , String . valueOf ( batchSize ) ) ; props . setProperty ( KafkaConstants . KEY_DESERIALIZER_CLASS_CONFIG , conf . keyDeserializer . getKeyClass ( ) ) ; props . setProperty ( KafkaConstants . VALUE_DESERIALIZER_CLASS_CONFIG , conf . valueDeserializer . getValueClass ( ) ) ; props . setProperty ( KafkaConstants . CONFLUENT_SCHEMA_REGISTRY_URL_CONFIG , StringUtils . join ( conf . dataFormatConfig . schemaRegistryUrls , "," ) ) ; props . setProperty ( KafkaConstants . AUTO_COMMIT_OFFEST , "false" ) ; if ( context . isPreview ( ) ) { props . setProperty ( KafkaConstants . AUTO_OFFSET_RESET_CONFIG , KafkaConstants . AUTO_OFFSET_RESET_PREVIEW_VALUE ) ; } return props ; }


Input: Checks whether any tables have had partitioning turned off or not and updates the partition map appropriately [CODESPLIT] private void handlePartitioningTurnedOffOrOn ( SortedSetMultimap < TableContext , TableRuntimeContext > reconstructedPartitions ) { for ( TableContext tableContext : reconstructedPartitions . keySet ( ) ) { final SortedSet < TableRuntimeContext > partitions = reconstructedPartitions . get ( tableContext ) ; final TableRuntimeContext lastPartition = partitions . last ( ) ; final TableContext sourceTableContext = lastPartition . getSourceTableContext ( ) ; Utils . checkState ( sourceTableContext . equals ( tableContext ) , String . format ( "Source table context for %s should match TableContext map key of %s" , lastPartition . getDescription ( ) , tableContext . getQualifiedName ( ) ) ) ; final boolean partitioningTurnedOff = lastPartition . isPartitioned ( ) && sourceTableContext . getPartitioningMode ( ) == PartitioningMode . DISABLED ; final boolean partitioningTurnedOn = ! lastPartition . isPartitioned ( ) && sourceTableContext . isPartitionable ( ) && sourceTableContext . getPartitioningMode ( ) != PartitioningMode . DISABLED ; if ( ! partitioningTurnedOff && ! partitioningTurnedOn ) { continue ; } final Map < String , String > nextStartingOffsets = new HashMap <> ( ) ; final Map < String , String > nextMaxOffsets = new HashMap <> ( ) ; final int newPartitionSequence = lastPartition . getPartitionSequence ( ) > 0 ? lastPartition . getPartitionSequence ( ) + 1 : 1 ; if ( partitioningTurnedOff ) { LOG . info ( "Table {} has switched from partitioned to non-partitioned; partition sequence {} will be the last (with" + " no max offsets)" , sourceTableContext . getQualifiedName ( ) , newPartitionSequence ) ; lastPartition . getStartingPartitionOffsets ( ) . forEach ( ( col , off ) -> { String basedOnStartOffset = lastPartition . generateNextPartitionOffset ( col , off ) ; nextStartingOffsets . put ( col , basedOnStartOffset ) ; } ) ; } else if ( partitioningTurnedOn ) { lastPartition . getStartingPartitionOffsets ( ) . forEach ( ( col , off ) -> { String basedOnStoredOffset = lastPartition . getInitialStoredOffsets ( ) . get ( col ) ; nextStartingOffsets . put ( col , basedOnStoredOffset ) ; } ) ; nextStartingOffsets . forEach ( ( col , off ) -> nextMaxOffsets . put ( col , lastPartition . generateNextPartitionOffset ( col , off ) ) ) ; if ( ! reconstructedPartitions . remove ( sourceTableContext , lastPartition ) ) { throw new IllegalStateException ( String . format ( "Failed to remove partition %s for table %s in switching partitioning from off to on" , lastPartition . getDescription ( ) , sourceTableContext . getQualifiedName ( ) ) ) ; } LOG . info ( "Table {} has switched from non-partitioned to partitioned; using last stored offsets as the starting" + " offsets for the new partition {}" , sourceTableContext . getQualifiedName ( ) , newPartitionSequence ) ; } final TableRuntimeContext nextPartition = new TableRuntimeContext ( sourceTableContext , lastPartition . isUsingNonIncrementalLoad ( ) , ( lastPartition . isPartitioned ( ) && ! partitioningTurnedOff ) || partitioningTurnedOn , newPartitionSequence , nextStartingOffsets , nextMaxOffsets ) ; reconstructedPartitions . put ( sourceTableContext , nextPartition ) ; } }


Input: Basically acquires more tables for the current thread to work on . The maximum a thread can hold is upper bounded to the value the thread number was allocated in { [CODESPLIT] @ VisibleForTesting void acquireTableAsNeeded ( int threadNumber ) throws InterruptedException { if ( ! getOwnedTablesQueue ( ) . isEmpty ( ) && batchTableStrategy == BatchTableStrategy . SWITCH_TABLES ) { final TableRuntimeContext lastOwnedPartition = getOwnedTablesQueue ( ) . pollLast ( ) ; if ( getTableContextMap ( ) . containsValue ( lastOwnedPartition . getSourceTableContext ( ) ) ) { sharedAvailableTablesQueue . offer ( lastOwnedPartition ) ; } TableContext lastOwnedTable = lastOwnedPartition . getSourceTableContext ( ) ; // need to cycle off all partitions from the same table to the end of the queue TableRuntimeContext first = sharedAvailableTablesQueue . peek ( ) ; while ( first != null && first . getSourceTableContext ( ) . equals ( lastOwnedTable ) && ! first . equals ( lastOwnedPartition ) ) { if ( LOG . isDebugEnabled ( ) ) { LOG . debug ( "Moving partition {} to end of shared queue to comply with BatchTableStrategy of {}" , first . getDescription ( ) , batchTableStrategy . getLabel ( ) ) ; } // poll() should never return null since it is actually returning 'first' as we want to move it from the head // of the queue, that's why we are calling offer, to basically remove it from the head but keep it in the queue TableRuntimeContext toMove = sharedAvailableTablesQueue . poll ( ) ; sharedAvailableTablesQueue . offer ( toMove ) ; // Get the new head of the queue first = sharedAvailableTablesQueue . peek ( ) ; } } if ( getOwnedTablesQueue ( ) . isEmpty ( ) ) { TableRuntimeContext head = sharedAvailableTablesQueue . poll ( ) ; if ( head != null ) { offerToOwnedTablesQueue ( head , threadNumber ) ; } } partitionFirstSharedQueueItemIfNeeded ( ) ; }


Input: <p > Examines the first item ( head ) im the shared partition queue and adds a new partition if appropriate< / p > <p > A new partition will be created if the number of partitions for the head item s table is still less than the maximum and that table itself is partitionable< / p > [CODESPLIT] @ VisibleForTesting void partitionFirstSharedQueueItemIfNeeded ( ) { final TableRuntimeContext headPartition = getOwnedTablesQueue ( ) . peek ( ) ; if ( headPartition != null ) { synchronized ( partitionStateLock ) { keepPartitioningIfNeeded ( headPartition ) ; } } else if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "No item at head of shared partition queue" ) ; } }


Input: Return the next table to work on for the current thread ( Will not return null ) Deque the current element from head of the queue and put it back at the tail to queue . [CODESPLIT] public TableRuntimeContext nextTable ( int threadNumber ) throws InterruptedException { synchronized ( partitionStateLock ) { acquireTableAsNeeded ( threadNumber ) ; final TableRuntimeContext partition = getOwnedTablesQueue ( ) . pollFirst ( ) ; if ( partition != null ) { offerToOwnedTablesQueue ( partition , threadNumber ) ; } return partition ; } }


Input: Each { [CODESPLIT] public void reportDataOrNoMoreData ( TableRuntimeContext tableRuntimeContext , int recordCount , int batchSize , boolean resultSetEndReached , AtomicBoolean tableFinished , AtomicBoolean schemaFinished , List < String > schemaFinishedTables ) { final TableContext sourceContext = tableRuntimeContext . getSourceTableContext ( ) ; // When we see a table with data, we mark isNoMoreDataEventGeneratedAlready to false // so we can generate event again if we don't see data from all tables. if ( recordCount > 0 ) { isNoMoreDataEventGeneratedAlready = false ; tablesWithNoMoreData . remove ( tableRuntimeContext . getSourceTableContext ( ) ) ; remainingSchemasToTableContexts . put ( sourceContext . getSchema ( ) , sourceContext ) ; completedSchemasToTableContexts . remove ( sourceContext . getSchema ( ) , sourceContext ) ; } // we need to account for the activeRuntimeContexts here // if there are still other active contexts in process, then this should do "nothing" // if there are not other contexts, we need to figure out what the highest offset completed by the last batch was final boolean noMoreData = recordCount == 0 || resultSetEndReached ; if ( noMoreData ) { tableRuntimeContext . setMarkedNoMoreData ( true ) ; } if ( recordCount > 0 ) { maxPartitionWithDataPerTable . put ( sourceContext , tableRuntimeContext . getPartitionSequence ( ) ) ; } boolean tableExhausted = removePartitionIfNeeded ( tableRuntimeContext ) ; if ( noMoreData ) { if ( tableExhausted ) { synchronized ( this ) { if ( LOG . isDebugEnabled ( ) ) { LOG . debug ( "Table {} exhausted" , sourceContext . getQualifiedName ( ) ) ; } final boolean newlyFinished = tablesWithNoMoreData . add ( sourceContext ) ; if ( newlyFinished && tableFinished != null ) { tableFinished . set ( true ) ; } final boolean remainingSchemaChanged = remainingSchemasToTableContexts . remove ( sourceContext . getSchema ( ) , sourceContext ) ; completedSchemasToTableContexts . put ( sourceContext . getSchema ( ) , sourceContext ) ; if ( remainingSchemaChanged && remainingSchemasToTableContexts . get ( sourceContext . getSchema ( ) ) . isEmpty ( ) && schemaFinished != null ) { schemaFinished . set ( true ) ; if ( schemaFinishedTables != null ) { completedSchemasToTableContexts . get ( sourceContext . getSchema ( ) ) . forEach ( t -> schemaFinishedTables . add ( t . getTableName ( ) ) ) ; } } } } } if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "Just released table {}; Number of Tables With No More Data {}" , tableRuntimeContext . getDescription ( ) , tablesWithNoMoreData . size ( ) ) ; } }


Input: Used by the main thread { [CODESPLIT] public synchronized boolean shouldGenerateNoMoreDataEvent ( ) { boolean noMoreData = ( ! isNoMoreDataEventGeneratedAlready && tablesWithNoMoreData . size ( ) == tableContextMap . size ( ) ) ; if ( noMoreData ) { isNoMoreDataEventGeneratedAlready = true ; } return noMoreData ; }


Input: / * Captures valid lines in currentLine and corresponding fields in fieldsFromLogLine . Captures stack traces if any in the argument stackTrace [CODESPLIT] private int readAhead ( Map < String , Field > fieldsFromLogLine , StringBuilder stackTrace ) throws DataParserException , IOException { StringBuilder multilineLog = new StringBuilder ( ) ; int read = readLine ( multilineLog ) ; int numberOfLinesRead = 0 ; while ( read > - 1 ) { try { Map < String , Field > stringFieldMap = parseLogLine ( multilineLog ) ; fieldsFromLogLine . putAll ( stringFieldMap ) ; currentLine . append ( multilineLog ) ; //If the line can be parsed successfully, do not read further //This line will be used in the current record if this is the first line being read //or stored for the next round if there is a line from the previous round. break ; } catch ( DataParserException e ) { //is this the first line being read? Yes -> throw exception if ( previousLine . length ( ) == 0 || maxStackTraceLines == - 1 ) { throw e ; } //otherwise read until we get a line that matches pattern if ( numberOfLinesRead < maxStackTraceLines ) { if ( numberOfLinesRead != 0 ) { stackTrace . append ( "\n" ) ; } stackTrace . append ( multilineLog . toString ( ) ) ; } numberOfLinesRead ++ ; multilineLog . setLength ( 0 ) ; read = readLine ( multilineLog ) ; } } return read ; }


Input: returns the reader line length the StringBuilder has up to maxObjectLen chars [CODESPLIT] int readLine ( StringBuilder sb ) throws IOException { int c = reader . read ( ) ; int count = ( c == - 1 ) ? - 1 : 0 ; while ( c > - 1 && ! isOverMaxObjectLen ( count ) && ! checkEolAndAdjust ( c ) ) { count ++ ; sb . append ( ( char ) c ) ; c = reader . read ( ) ; } if ( isOverMaxObjectLen ( count ) ) { sb . setLength ( sb . length ( ) - 1 ) ; while ( c > - 1 && c != ' ' && c != ' ' ) { count ++ ; c = reader . read ( ) ; } checkEolAndAdjust ( c ) ; } return count ; }


Input: returns fresh UsageTimer just reset to zero accumulated time [CODESPLIT] public UsageTimer roll ( ) { int multiplier ; synchronized ( this ) { multiplier = getMultiplier ( ) ; changeMultiplier ( - multiplier ) ; //stopAll; } return new UsageTimer ( ) . setName ( getName ( ) ) . changeMultiplier ( multiplier ) ; }


Input: Enable Control Hub on this Data Collector . [CODESPLIT] public static void enableDPM ( DPMInfoJson dpmInfo , Context context ) throws IOException { Utils . checkNotNull ( dpmInfo , "DPMInfo" ) ; String dpmBaseURL = normalizeDpmBaseURL ( dpmInfo . getBaseURL ( ) ) ; // Since we support enabling/Disabling DPM, first check if token already exists for the given DPM URL. // If token exists skip first 3 steps String currentDPMBaseURL = context . configuration . get ( RemoteSSOService . DPM_BASE_URL_CONFIG , "" ) ; String currentAppAuthToken = context . configuration . get ( RemoteSSOService . SECURITY_SERVICE_APP_AUTH_TOKEN_CONFIG , "" ) . trim ( ) ; if ( ! currentDPMBaseURL . equals ( dpmBaseURL ) || currentAppAuthToken . length ( ) == 0 ) { // 1. Login to DPM to get user auth token String userAuthToken = retrieveUserToken ( dpmBaseURL , dpmInfo . getUserID ( ) , dpmInfo . getUserPassword ( ) ) ; String appAuthToken = null ; // 2. Create Data Collector application token Response response = null ; try { Map < String , Object > newComponentJson = new HashMap <> ( ) ; newComponentJson . put ( "organization" , dpmInfo . getOrganization ( ) ) ; newComponentJson . put ( "componentType" , "dc" ) ; newComponentJson . put ( "numberOfComponents" , 1 ) ; newComponentJson . put ( "active" , true ) ; response = ClientBuilder . newClient ( ) . target ( dpmBaseURL + "/security/rest/v1/organization/" + dpmInfo . getOrganization ( ) + "/components" ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . header ( SSOConstants . X_USER_AUTH_TOKEN , userAuthToken ) . put ( Entity . json ( newComponentJson ) ) ; if ( response . getStatus ( ) != Response . Status . CREATED . getStatusCode ( ) ) { throw new RuntimeException ( Utils . format ( "DPM Create Application Token failed, status code '{}': {}" , response . getStatus ( ) , response . readEntity ( String . class ) ) ) ; } List < Map < String , Object > > newComponent = response . readEntity ( new GenericType < List < Map < String , Object > > > ( ) { } ) ; if ( newComponent . size ( ) > 0 ) { appAuthToken = ( String ) newComponent . get ( 0 ) . get ( "fullAuthToken" ) ; } else { throw new RuntimeException ( "DPM Create Application Token failed: No token data from DPM Server." ) ; } } finally { if ( response != null ) { response . close ( ) ; } // Logout from DPM logout ( dpmBaseURL , userAuthToken ) ; } // 3. Update App Token file updateTokenFile ( context , appAuthToken ) ; } // 4. Update dpm.properties file updateDpmProperties ( context , dpmBaseURL , dpmInfo . getLabels ( ) , true ) ; }


Input: Disable Control Hub on this Data Collector - with explicit login . [CODESPLIT] public static void disableDPM ( String username , String password , String organizationId , Context context ) throws IOException { String dpmBaseURL = normalizeDpmBaseURL ( context . configuration . get ( RemoteSSOService . DPM_BASE_URL_CONFIG , "" ) ) ; String userToken = retrieveUserToken ( dpmBaseURL , username , password ) ; try { disableDPM ( userToken , organizationId , context ) ; } finally { logout ( dpmBaseURL , userToken ) ; } }


Input: Disable Control Hub on this Data Collector - using existing auth token . [CODESPLIT] public static void disableDPM ( String userAuthToken , String organizationId , Context context ) throws IOException { // check if DPM enabled if ( ! context . runtimeInfo . isDPMEnabled ( ) ) { throw new RuntimeException ( "disableDPM is supported only when DPM is enabled" ) ; } String dpmBaseURL = normalizeDpmBaseURL ( context . configuration . get ( RemoteSSOService . DPM_BASE_URL_CONFIG , "" ) ) ; String componentId = context . runtimeInfo . getId ( ) ; // 2. Deactivate Data Collector System Component Response response = null ; try { response = ClientBuilder . newClient ( ) . target ( dpmBaseURL + "/security/rest/v1/organization/" + organizationId + "/components/deactivate" ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . header ( SSOConstants . X_USER_AUTH_TOKEN , userAuthToken ) . header ( SSOConstants . X_REST_CALL , true ) . post ( Entity . json ( ImmutableList . of ( componentId ) ) ) ; if ( response . getStatus ( ) != Response . Status . OK . getStatusCode ( ) ) { throw new RuntimeException ( Utils . format ( " Deactivate Data Collector System Component from DPM failed, status code '{}': {}" , response . getStatus ( ) , response . readEntity ( String . class ) ) ) ; } } finally { if ( response != null ) { response . close ( ) ; } } // 3. Delete Data Collector System Component try { response = ClientBuilder . newClient ( ) . target ( dpmBaseURL + "/security/rest/v1/organization/" + organizationId + "/components/delete" ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . header ( SSOConstants . X_USER_AUTH_TOKEN , userAuthToken ) . header ( SSOConstants . X_REST_CALL , true ) . post ( Entity . json ( ImmutableList . of ( componentId ) ) ) ; if ( response . getStatus ( ) != Response . Status . OK . getStatusCode ( ) ) { throw new RuntimeException ( Utils . format ( " Deactivate Data Collector System Component from DPM failed, status code '{}': {}" , response . getStatus ( ) , response . readEntity ( String . class ) ) ) ; } } finally { if ( response != null ) { response . close ( ) ; } } // 4. Delete from Job Runner SDC list try { response = ClientBuilder . newClient ( ) . target ( dpmBaseURL + "/jobrunner/rest/v1/sdc/" + componentId ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . header ( SSOConstants . X_USER_AUTH_TOKEN , userAuthToken ) . header ( SSOConstants . X_REST_CALL , true ) . delete ( ) ; if ( response . getStatus ( ) != Response . Status . OK . getStatusCode ( ) ) { throw new RuntimeException ( Utils . format ( "Delete from DPM Job Runner SDC list failed, status code '{}': {}" , response . getStatus ( ) , response . readEntity ( String . class ) ) ) ; } } finally { if ( response != null ) { response . close ( ) ; } } // 5. Update App Token file updateTokenFile ( context , "" ) ; // 4. Update dpm.properties file updateDpmProperties ( context , dpmBaseURL , null , false ) ; }


Input: Normalize Control Hub URL - primarily drop training slash . [CODESPLIT] private static String normalizeDpmBaseURL ( String url ) { if ( url . endsWith ( "/" ) ) { url = url . substring ( 0 , url . length ( ) - 1 ) ; } return url ; }


Input: Login user and retrieve authentication token . [CODESPLIT] private static String retrieveUserToken ( String url , String username , String password ) { Response response = null ; try { Map < String , String > loginJson = new HashMap <> ( ) ; loginJson . put ( "userName" , username ) ; loginJson . put ( "password" , password ) ; response = ClientBuilder . newClient ( ) . target ( url + "/security/public-rest/v1/authentication/login" ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . post ( Entity . json ( loginJson ) ) ; if ( response . getStatus ( ) != Response . Status . OK . getStatusCode ( ) ) { throw new RuntimeException ( Utils . format ( "DPM Login failed, status code '{}': {}" , response . getStatus ( ) , response . readEntity ( String . class ) ) ) ; } } finally { if ( response != null ) { response . close ( ) ; } } return response . getHeaderString ( SSOConstants . X_USER_AUTH_TOKEN ) ; }


Input: Logout given token . [CODESPLIT] private static void logout ( String dpmBaseURL , String userAuthToken ) { Response response = null ; try { response = ClientBuilder . newClient ( ) . target ( dpmBaseURL + "/security/_logout" ) . register ( new CsrfProtectionFilter ( "CSRF" ) ) . request ( ) . header ( SSOConstants . X_USER_AUTH_TOKEN , userAuthToken ) . cookie ( SSOConstants . AUTHENTICATION_COOKIE_PREFIX + "LOGIN" , userAuthToken ) . get ( ) ; } finally { if ( response != null ) { response . close ( ) ; } } }


Input: Update token file with the SDC access token . [CODESPLIT] private static void updateTokenFile ( Context context , String appAuthToken ) throws IOException { File tokenFile = context . tokenFilePath == null ? new File ( context . runtimeInfo . getConfigDir ( ) , APP_TOKEN_FILE ) : new File ( context . tokenFilePath ) ; DataStore dataStore = new DataStore ( tokenFile ) ; try ( OutputStream os = dataStore . getOutputStream ( ) ) { IOUtils . write ( appAuthToken , os ) ; dataStore . commit ( os ) ; } finally { dataStore . release ( ) ; } }


Input: Update dpm . properties file with new configuration . [CODESPLIT] private static void updateDpmProperties ( Context context , String dpmBaseURL , List < String > labels , boolean enableSch ) { if ( context . skipUpdatingDpmProperties ) { return ; } try { FileBasedConfigurationBuilder < PropertiesConfiguration > builder = new FileBasedConfigurationBuilder <> ( PropertiesConfiguration . class ) . configure ( new Parameters ( ) . properties ( ) . setFileName ( context . runtimeInfo . getConfigDir ( ) + "/dpm.properties" ) . setThrowExceptionOnMissing ( true ) . setListDelimiterHandler ( new DefaultListDelimiterHandler ( ' ' ) ) . setIncludesAllowed ( false ) ) ; PropertiesConfiguration config = null ; config = builder . getConfiguration ( ) ; config . setProperty ( RemoteSSOService . DPM_ENABLED , Boolean . toString ( enableSch ) ) ; config . setProperty ( RemoteSSOService . DPM_BASE_URL_CONFIG , dpmBaseURL ) ; config . setProperty ( RemoteSSOService . SECURITY_SERVICE_APP_AUTH_TOKEN_CONFIG , APP_TOKEN_FILE_PROP_VAL ) ; if ( labels != null && labels . size ( ) > 0 ) { config . setProperty ( RemoteEventHandlerTask . REMOTE_JOB_LABELS , StringUtils . join ( labels , ' ' ) ) ; } else { config . setProperty ( RemoteEventHandlerTask . REMOTE_JOB_LABELS , "" ) ; } builder . save ( ) ; } catch ( ConfigurationException e ) { throw new RuntimeException ( Utils . format ( "Updating dpm.properties file failed: {}" , e . getMessage ( ) ) , e ) ; } }