Input: Create record and add it to { [CODESPLIT] @ Override public void createAndAddRecord ( ResultSet rs , TableRuntimeContext tableRuntimeContext , BatchContext batchContext ) throws SQLException , StageException { ResultSetMetaData md = rs . getMetaData ( ) ; LinkedHashMap < String , Field > fields = jdbcUtil . resultSetToFields ( rs , commonSourceConfigBean , errorRecordHandler , tableJdbcConfigBean . unknownTypeAction , recordHeader , DatabaseVendor . SQL_SERVER ) ; Map < String , String > columnOffsets = new HashMap <> ( ) ; // Generate Offset includes primary keys, sys_change_version, and sys_change_operation for ( String key : tableRuntimeContext . getSourceTableContext ( ) . getOffsetColumns ( ) ) { String value = rs . getString ( key ) ; if ( Strings . isNullOrEmpty ( value ) ) { value = fields . get ( key ) != null ? fields . get ( key ) . getValueAsString ( ) : "" ; } columnOffsets . put ( key , value ) ; } columnOffsets . put ( SYS_CHANGE_OPERATION , rs . getString ( SYS_CHANGE_OPERATION ) ) ; String offsetFormat = OffsetQueryUtil . getOffsetFormat ( columnOffsets ) ; Record record = context . createRecord ( tableRuntimeContext . getQualifiedName ( ) + "::" + offsetFormat ) ; record . set ( Field . createListMap ( fields ) ) ; //Set Column Headers jdbcUtil . setColumnSpecificHeaders ( record , Collections . singleton ( tableRuntimeContext . getSourceTableContext ( ) . getTableName ( ) ) , md , JDBC_NAMESPACE_HEADER ) ; //Set Operation Headers int op = MSOperationCode . convertToJDBCCode ( rs . getString ( SYS_CHANGE_OPERATION ) ) ; record . getHeader ( ) . setAttribute ( OperationType . SDC_OPERATION_TYPE , String . valueOf ( op ) ) ; for ( String fieldName : recordHeader ) { record . getHeader ( ) . setAttribute ( JDBC_NAMESPACE_HEADER + fieldName , rs . getString ( fieldName ) != null ? rs . getString ( fieldName ) : "NULL" ) ; } int columns = rs . getMetaData ( ) . getColumnCount ( ) ; if ( fields . size ( ) != columns ) { errorRecordHandler . onError ( JdbcErrors . JDBC_35 , fields . size ( ) , columns ) ; return ; // Don't output this record. } else { batchContext . getBatchMaker ( ) . addRecord ( record ) ; } offsets . put ( tableRuntimeContext . getOffsetKey ( ) , offsetFormat ) ; }


Input: When pullMap is called the caller should have consumed the opening tag for the record [CODESPLIT] private Field pullMap ( XMLEventReader reader ) throws StageException , XMLStreamException { LinkedHashMap < String , Field > map = new LinkedHashMap <> ( ) ; String type = null ; String fieldValue = null ; while ( reader . hasNext ( ) ) { XMLEvent event = reader . nextEvent ( ) ; if ( event . isStartElement ( ) ) { if ( event . asStartElement ( ) . getName ( ) . getLocalPart ( ) . equals ( TYPE ) ) { // Move to content event = reader . nextEvent ( ) ; type = event . asCharacters ( ) . getData ( ) . toLowerCase ( ) ; // Consume closing tag reader . nextEvent ( ) ; } else { String fieldName = event . asStartElement ( ) . getName ( ) . getLocalPart ( ) ; Attribute attr = event . asStartElement ( ) . getAttributeByName ( XSI_TYPE ) ; if ( attr != null && attr . getValue ( ) . equals ( S_OBJECT ) ) { // Element is a nested record map . put ( fieldName , pullMap ( reader ) ) ; } else { event = reader . nextEvent ( ) ; fieldValue = null ; switch ( event . getEventType ( ) ) { case XMLEvent . START_ELEMENT : // Element is a nested list of records // Advance over <done>, <queryLocator> to record list while ( ! ( event . isStartElement ( ) && event . asStartElement ( ) . getName ( ) . getLocalPart ( ) . equals ( RECORDS ) ) ) { event = reader . nextEvent ( ) ; } // Read record list List < Field > recordList = new ArrayList <> ( ) ; while ( event . isStartElement ( ) && event . asStartElement ( ) . getName ( ) . getLocalPart ( ) . equals ( RECORDS ) ) { recordList . add ( pullMap ( reader ) ) ; event = reader . nextEvent ( ) ; } map . put ( fieldName , Field . create ( recordList ) ) ; break ; case XMLEvent . CHARACTERS : // Element is a field value fieldValue = event . asCharacters ( ) . getData ( ) ; // Consume closing tag reader . nextEvent ( ) ; // Intentional fall through to next case! case XMLEvent . END_ELEMENT : // Create the SDC field if ( type == null ) { throw new StageException ( Errors . FORCE_38 ) ; } // Is this a relationship to another object? com . sforce . soap . partner . Field sfdcField = metadataCache . get ( type ) . getFieldFromRelationship ( fieldName ) ; if ( sfdcField != null ) { // See if we already added fields from the related record if ( map . get ( fieldName ) != null ) { // We already created this node - don't overwrite it! sfdcField = null ; } } else { sfdcField = getFieldMetadata ( type , fieldName ) ; } if ( sfdcField != null ) { Field field = createField ( fieldValue , sfdcField ) ; if ( conf . createSalesforceNsHeaders ) { setHeadersOnField ( field , getFieldMetadata ( type , fieldName ) ) ; } map . put ( fieldName , field ) ; } break ; default : throw new StageException ( Errors . FORCE_41 , event . getEventType ( ) ) ; } } } } else if ( event . isEndElement ( ) ) { // Done with record return Field . createListMap ( map ) ; } } throw new StageException ( Errors . FORCE_39 ) ; }


Input: / * Terminates the request with an HTTP Unauthorized response [CODESPLIT] protected Authentication returnUnauthorized ( HttpServletRequest httpReq , HttpServletResponse httpRes , String principalId , String logMessageTemplate ) throws ServerAuthException { return returnUnauthorized ( httpReq , httpRes , UNAUTHORIZED_JSON , principalId , logMessageTemplate ) ; }


Input: Generate schema for given field and optionally wrap it in union with null if configured . [CODESPLIT] private Schema . Field schemaFieldForType ( String fieldPath , Record record , String fieldName , Field field ) throws OnRecordErrorException { Schema simpleSchema = simpleSchemaForType ( fieldPath , record , field ) ; Schema finalSchema = simpleSchema ; // If Nullable check box was selected, wrap the whole schema in union with null if ( getConfig ( ) . avroNullableFields ) { finalSchema = Schema . createUnion ( ImmutableList . of ( Schema . create ( Schema . Type . NULL ) , simpleSchema ) ) ; } return new Schema . Field ( fieldName , finalSchema , null , getDefaultValue ( simpleSchema ) ) ; }


Input: Generates complex schema for given field that will include optional union with null and potentially default value as well . Particularly useful to generate nested structures . [CODESPLIT] private Schema complexSchemaForType ( String fieldPath , Record record , Field field ) throws OnRecordErrorException { Schema simpleSchema = simpleSchemaForType ( fieldPath , record , field ) ; Schema finalSchema = simpleSchema ; if ( getConfig ( ) . avroNullableFields ) { finalSchema = Schema . createUnion ( ImmutableList . of ( Schema . create ( Schema . Type . NULL ) , simpleSchema ) ) ; } JsonNode defaultValue = getDefaultValue ( simpleSchema ) ; if ( defaultValue != null ) { finalSchema . addProp ( "defaultValue" , defaultValue ) ; } return finalSchema ; }


Input: Generate simple schema for given field - it will never contain union nor a default value . Particularly useful for generating Schema . Field as this will not convert simple string to { type : string defaultValue : something } which is hard to undo . [CODESPLIT] private Schema simpleSchemaForType ( String fieldPath , Record record , Field field ) throws OnRecordErrorException { switch ( field . getType ( ) ) { // Primitive types case BOOLEAN : return Schema . create ( Schema . Type . BOOLEAN ) ; case INTEGER : return Schema . create ( Schema . Type . INT ) ; case LONG : return Schema . create ( Schema . Type . LONG ) ; case FLOAT : return Schema . create ( Schema . Type . FLOAT ) ; case DOUBLE : return Schema . create ( Schema . Type . DOUBLE ) ; case STRING : return Schema . create ( Schema . Type . STRING ) ; case BYTE_ARRAY : return Schema . create ( Schema . Type . BYTES ) ; // Logical types case DECIMAL : int precision = getDecimalScaleOrPrecision ( record , field , getConfig ( ) . precisionAttribute , getConfig ( ) . defaultPrecision , 1 ) ; int scale = getDecimalScaleOrPrecision ( record , field , getConfig ( ) . scaleAttribute , getConfig ( ) . defaultScale , 0 ) ; Schema decimalSchema = Schema . create ( Schema . Type . BYTES ) ; decimalSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_DECIMAL ) ; decimalSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE_ATTR_PRECISION , new IntNode ( precision ) ) ; decimalSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE_ATTR_SCALE , new IntNode ( scale ) ) ; return decimalSchema ; case DATE : Schema dateSchema = Schema . create ( Schema . Type . INT ) ; dateSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_DATE ) ; return dateSchema ; case TIME : Schema timeSchema = Schema . create ( Schema . Type . INT ) ; timeSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_TIME_MILLIS ) ; return timeSchema ; case DATETIME : Schema dateTimeSchema = Schema . create ( Schema . Type . LONG ) ; dateTimeSchema . addProp ( AvroTypeUtil . LOGICAL_TYPE , AvroTypeUtil . LOGICAL_TYPE_TIMESTAMP_MILLIS ) ; return dateTimeSchema ; // Complex types case LIST : // In avro list must be of the same type - which is not true with our records // We can't generate the list type from empty list if ( field . getValueAsList ( ) . isEmpty ( ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0006 , fieldPath ) ; } // And all items in the list must have the same schema Schema itemSchema = null ; int index = 0 ; for ( Field listItem : field . getValueAsList ( ) ) { Schema currentListItemSchema = complexSchemaForType ( fieldPath + "[" + index + "]" , record , listItem ) ; if ( itemSchema == null ) { itemSchema = currentListItemSchema ; } else if ( ! itemSchema . equals ( currentListItemSchema ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0005 , fieldPath , itemSchema , currentListItemSchema ) ; } index ++ ; } return Schema . createArray ( itemSchema ) ; case MAP : case LIST_MAP : // In avro maps must have key of string (same as us) and value must be the same for all items - which // is different then our records. // We can't generate the map value type from null or empty map if ( field . getValueAsMap ( ) == null || field . getValueAsMap ( ) . isEmpty ( ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0008 , fieldPath ) ; } // And all values in the map must be the same Schema mapSchema = null ; for ( Map . Entry < String , Field > item : field . getValueAsMap ( ) . entrySet ( ) ) { Schema currentListItemSchema = complexSchemaForType ( fieldPath + "/" + item . getKey ( ) , record , item . getValue ( ) ) ; if ( mapSchema == null ) { mapSchema = currentListItemSchema ; } else if ( ! mapSchema . equals ( currentListItemSchema ) ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0007 , fieldPath , mapSchema , currentListItemSchema ) ; } } return Schema . createMap ( mapSchema ) ; // Types that does not have direct equivalent in Avro case SHORT : if ( getConfig ( ) . avroExpandTypes ) { return Schema . create ( Schema . Type . INT ) ; } // fall through case CHAR : if ( getConfig ( ) . avroExpandTypes ) { return Schema . create ( Schema . Type . STRING ) ; } // fall through // Not supported data types: case BYTE : case FILE_REF : default : throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0002 , field . getType ( ) ) ; } }


Input: Resolve parameters of decimal type . [CODESPLIT] private int getDecimalScaleOrPrecision ( Record record , Field field , String attributeName , int defaultValue , int minAllowed ) throws OnRecordErrorException { int finalValue = - 1 ; // Invalid value // Firstly try the field attribute String stringValue = field . getAttribute ( attributeName ) ; if ( ! StringUtils . isEmpty ( stringValue ) ) { finalValue = Integer . valueOf ( stringValue ) ; } // If it's invalid, then use the default value if ( finalValue < minAllowed ) { finalValue = defaultValue ; } // If even the default value is invalid, then send the record to error if ( finalValue < minAllowed ) { throw new OnRecordErrorException ( record , Errors . SCHEMA_GEN_0004 , finalValue , field ) ; } return finalValue ; }


Input: Returns default value for given field or null if no default value should be used . [CODESPLIT] private JsonNode getDefaultValue ( Schema schema ) { if ( getConfig ( ) . avroNullableFields && getConfig ( ) . avroDefaultNullable ) { return NullNode . getInstance ( ) ; } if ( ! getConfig ( ) . avroNullableFields && defaultValuesForTypes . containsKey ( schema . getType ( ) ) ) { return defaultValuesForTypes . get ( schema . getType ( ) ) ; } return null ; }


Input: Creates a gauge if it is already not . This is done only once for the stage [CODESPLIT] @ SuppressWarnings ( "unchecked" ) public static synchronized void initMetricsIfNeeded ( ProtoConfigurableEntity . Context context ) { Gauge < Map < String , Object > > gauge = context . getGauge ( fileStatisticGaugeName ( context ) ) ; if ( gauge == null ) { gauge = context . createGauge ( fileStatisticGaugeName ( context ) , Comparator . comparing ( GAUGE_MAP_ORDERING :: get ) ) ; Map < String , Object > gaugeStatistics = gauge . getValue ( ) ; //File name is populated at the MetricEnabledWrapperStream. gaugeStatistics . put ( FileRefUtil . FILE , "" ) ; gaugeStatistics . put ( FileRefUtil . TRANSFER_THROUGHPUT , 0L ) ; gaugeStatistics . put ( FileRefUtil . SENT_BYTES , String . format ( FileRefUtil . BRACKETED_TEMPLATE , 0 , 0 ) ) ; gaugeStatistics . put ( FileRefUtil . REMAINING_BYTES , 0L ) ; gaugeStatistics . put ( FileRefUtil . COMPLETED_FILE_COUNT , 0L ) ; } Meter dataTransferMeter = context . getMeter ( FileRefUtil . TRANSFER_THROUGHPUT_METER ) ; if ( dataTransferMeter == null ) { context . createMeter ( FileRefUtil . TRANSFER_THROUGHPUT_METER ) ; } }


Input: This method is a simple wrapper that lets us find the NoSuchFileException if that was the cause . [CODESPLIT] public int compare ( WrappedFile path1 , WrappedFile path2 , boolean useLastModified ) { // why not just check if the file exists? Well, there is a possibility file gets moved/archived/deleted right after // that check. In that case we will still fail. So fail, and recover. try { if ( useLastModified && ! exists ( path2 ) ) { return 1 ; } return getComparator ( useLastModified ) . compare ( path1 , path2 ) ; } catch ( RuntimeException ex ) { Throwable cause = ex . getCause ( ) ; // Happens only in timestamp ordering. // Very unlikely this will happen, new file has to be added to the queue at the exact time when // the currentFile was consumed and archived while a new file has not yet been picked up for processing. // Ignore - we just add the new file, since this means this file is indeed newer // (else this would have been consumed and archived first) if ( cause != null && cause instanceof NoSuchFileException ) { LOG . debug ( "Starting file may have already been archived." , cause ) ; return 1 ; } LOG . warn ( "Error while comparing files" , ex ) ; throw ex ; } }


Input: Gets the group index of a named capture group at the specified index . If only one instance of the named group exists use index 0 . [CODESPLIT] public int indexOf ( String groupName , int index ) { int idx = - 1 ; if ( groupInfo . containsKey ( groupName ) ) { List < GroupInfo > list = groupInfo . get ( groupName ) ; idx = list . get ( index ) . groupIndex ( ) ; } return idx ; }


Input: Gets the names of all capture groups [CODESPLIT] public List < String > groupNames ( ) { if ( groupNames == null ) { groupNames = new ArrayList < String > ( groupInfo . keySet ( ) ) ; } return groupNames ; }


Input: Determines if the character at the specified position of a string is escaped [CODESPLIT] static private boolean isEscapedChar ( String s , int pos ) { return isSlashEscapedChar ( s , pos ) || isQuoteEscapedChar ( s , pos ) ; }


Input: Determines if the character at the specified position of a string is escaped with a backslash [CODESPLIT] static private boolean isSlashEscapedChar ( String s , int pos ) { // Count the backslashes preceding this position. If it's // even, there is no escape and the slashes are just literals. // If it's odd, one of the slashes (the last one) is escaping // the character at the given position. int numSlashes = 0 ; while ( pos > 0 && ( s . charAt ( pos - 1 ) == ' ' ) ) { pos -- ; numSlashes ++ ; } return numSlashes % 2 != 0 ; }


Input: Determines if the character at the specified position of a string is quote - escaped ( between \\ Q and \\ E ) [CODESPLIT] static private boolean isQuoteEscapedChar ( String s , int pos ) { boolean openQuoteFound = false ; boolean closeQuoteFound = false ; // find last non-escaped open-quote String s2 = s . substring ( 0 , pos ) ; int posOpen = pos ; while ( ( posOpen = s2 . lastIndexOf ( "\\Q" , posOpen - 1 ) ) != - 1 ) { if ( ! isSlashEscapedChar ( s2 , posOpen ) ) { openQuoteFound = true ; break ; } } if ( openQuoteFound ) { // search remainder of string (after open-quote) for a close-quote; // no need to check that it's slash-escaped because it can't be // (the escape character itself is part of the literal when quoted) if ( s2 . indexOf ( "\\E" , posOpen ) != - 1 ) { closeQuoteFound = true ; } } return openQuoteFound && ! closeQuoteFound ; }


Input: Determines if a string s character is within a regex character class [CODESPLIT] static private boolean isInsideCharClass ( String s , int pos ) { boolean openBracketFound = false ; boolean closeBracketFound = false ; // find last non-escaped open-bracket String s2 = s . substring ( 0 , pos ) ; int posOpen = pos ; while ( ( posOpen = s2 . lastIndexOf ( ' ' , posOpen - 1 ) ) != - 1 ) { if ( ! isEscapedChar ( s2 , posOpen ) ) { openBracketFound = true ; break ; } } if ( openBracketFound ) { // search remainder of string (after open-bracket) for a close-bracket String s3 = s . substring ( posOpen , pos ) ; int posClose = - 1 ; while ( ( posClose = s3 . indexOf ( ' ' , posClose + 1 ) ) != - 1 ) { if ( ! isEscapedChar ( s3 , posClose ) ) { closeBracketFound = true ; break ; } } } return openBracketFound && ! closeBracketFound ; }


Input: Determines if the parenthesis at the specified position of a string is for a non - capturing group which is one of the flag specifiers ( e . g . ( ?s ) or ( ?m ) or ( ? : pattern ) . If the parenthesis is followed by ? it must be a non - capturing group unless it s a named group ( which begins with ?< ) . Make sure not to confuse it with the lookbehind construct ( ?< = or ?<! ) . [CODESPLIT] static private boolean isNoncapturingParen ( String s , int pos ) { //int len = s.length(); boolean isLookbehind = false ; // code-coverage reports show that pos and the text to // check never exceed len in this class, so it's safe // to not test for it, which resolves uncovered branches // in Cobertura /*if (pos >= 0 && pos + 4 < len)*/ { String pre = s . substring ( pos , pos + 4 ) ; isLookbehind = pre . equals ( "(?<=" ) || pre . equals ( "(?<!" ) ; } return /*(pos >= 0 && pos + 2 < len) &&*/ s . charAt ( pos + 1 ) == ' ' && ( isLookbehind || s . charAt ( pos + 2 ) != ' ' ) ; }


Input: Counts the open - parentheses to the left of a string position excluding escaped parentheses [CODESPLIT] static private int countOpenParens ( String s , int pos ) { java . util . regex . Pattern p = java . util . regex . Pattern . compile ( "\\(" ) ; java . util . regex . Matcher m = p . matcher ( s . subSequence ( 0 , pos ) ) ; int numParens = 0 ; while ( m . find ( ) ) { // ignore parentheses inside character classes: [0-9()a-f] // which are just literals if ( isInsideCharClass ( s , m . start ( ) ) ) { continue ; } // ignore escaped parens if ( isEscapedChar ( s , m . start ( ) ) ) continue ; if ( ! isNoncapturingParen ( s , m . start ( ) ) ) { numParens ++ ; } } return numParens ; }


Input: Parses info on named capture groups from a pattern [CODESPLIT] static public Map < String , List < GroupInfo > > extractGroupInfo ( String namedPattern ) { Map < String , List < GroupInfo > > groupInfo = new LinkedHashMap < String , List < GroupInfo > > ( ) ; java . util . regex . Matcher matcher = NAMED_GROUP_PATTERN . matcher ( namedPattern ) ; while ( matcher . find ( ) ) { int pos = matcher . start ( ) ; // ignore escaped paren if ( isEscapedChar ( namedPattern , pos ) ) continue ; String name = matcher . group ( INDEX_GROUP_NAME ) ; int groupIndex = countOpenParens ( namedPattern , pos ) ; List < GroupInfo > list ; if ( groupInfo . containsKey ( name ) ) { list = groupInfo . get ( name ) ; } else { list = new ArrayList < GroupInfo > ( ) ; } list . add ( new GroupInfo ( groupIndex , pos ) ) ; groupInfo . put ( name , list ) ; } return groupInfo ; }


Input: Replaces strings matching a pattern with another string . If the string to be replaced is escaped with a slash it is skipped . [CODESPLIT] static private StringBuilder replace ( StringBuilder input , java . util . regex . Pattern pattern , String replacement ) { java . util . regex . Matcher m = pattern . matcher ( input ) ; while ( m . find ( ) ) { if ( isEscapedChar ( input . toString ( ) , m . start ( ) ) ) { continue ; } // since we're replacing the original string being matched, // we have to reset the matcher so that it searches the new // string input . replace ( m . start ( ) , m . end ( ) , replacement ) ; m . reset ( input ) ; } return input ; }


Input: Replaces referenced group names with the reference to the corresponding group index ( e . g . <b > <code > \ k&lt ; named > < / code > < / b > } to <b > <code > \ k2< / code > < / b > } ; <b > <code > $ { named } < / code > < / b > to <b > <code > $2< / code > < / b > } ) . This assumes the group names have already been parsed from the pattern . [CODESPLIT] private StringBuilder replaceGroupNameWithIndex ( StringBuilder input , java . util . regex . Pattern pattern , String prefix ) { java . util . regex . Matcher m = pattern . matcher ( input ) ; while ( m . find ( ) ) { if ( isEscapedChar ( input . toString ( ) , m . start ( ) ) ) { continue ; } int index = indexOf ( m . group ( INDEX_GROUP_NAME ) ) ; if ( index >= 0 ) { index ++ ; } else { throw new PatternSyntaxException ( "unknown group name" , input . toString ( ) , m . start ( INDEX_GROUP_NAME ) ) ; } // since we're replacing the original string being matched, // we have to reset the matcher so that it searches the new // string input . replace ( m . start ( ) , m . end ( ) , prefix + index ) ; m . reset ( input ) ; } return input ; }


Input: Builds a { @code java . util . regex . Pattern } from a given regular expression pattern ( which may contain named groups ) and flags [CODESPLIT] private java . util . regex . Pattern buildStandardPattern ( String namedPattern , Integer flags ) { // replace the named-group construct with left-paren but // make sure we're actually looking at the construct (ignore escapes) StringBuilder s = new StringBuilder ( namedPattern ) ; s = replace ( s , NAMED_GROUP_PATTERN , "(" ) ; s = replaceGroupNameWithIndex ( s , BACKREF_NAMED_GROUP_PATTERN , "\\" ) ; return java . util . regex . Pattern . compile ( s . toString ( ) , flags ) ; }


Input: Process a GET request for the specified resource . [CODESPLIT] @ Override public void doGet ( HttpServletRequest request , HttpServletResponse response ) { try { JsonGenerator jg = null ; String jsonpcb = null ; PrintWriter writer = null ; try { writer = response . getWriter ( ) ; // "callback" parameter implies JSONP outpout jsonpcb = request . getParameter ( CALLBACK_PARAM ) ; if ( jsonpcb != null ) { response . setContentType ( "application/javascript; charset=utf8" ) ; writer . write ( jsonpcb + "(" ) ; } else { response . setContentType ( "application/json; charset=utf8" ) ; } jg = jsonFactory . createGenerator ( writer ) ; jg . disable ( JsonGenerator . Feature . AUTO_CLOSE_TARGET ) ; jg . useDefaultPrettyPrinter ( ) ; jg . writeStartObject ( ) ; // query per mbean attribute String getmethod = request . getParameter ( "get" ) ; if ( getmethod != null ) { String [ ] splitStrings = getmethod . split ( "\\:\\:" ) ; if ( splitStrings . length != 2 ) { jg . writeStringField ( "result" , "ERROR" ) ; jg . writeStringField ( "message" , "query format is not as expected." ) ; jg . flush ( ) ; response . setStatus ( HttpServletResponse . SC_BAD_REQUEST ) ; return ; } listBeans ( jg , new ObjectName ( splitStrings [ 0 ] ) , splitStrings [ 1 ] , response ) ; return ; } // query per mbean String qry = request . getParameter ( "qry" ) ; if ( qry == null ) { qry = "*:*" ; } listBeans ( jg , new ObjectName ( qry ) , null , response ) ; } finally { if ( jg != null ) { jg . close ( ) ; } if ( jsonpcb != null ) { writer . write ( ");" ) ; } if ( writer != null ) { writer . close ( ) ; } } } catch ( IOException e ) { response . setStatus ( HttpServletResponse . SC_INTERNAL_SERVER_ERROR ) ; } catch ( MalformedObjectNameException e ) { response . setStatus ( HttpServletResponse . SC_BAD_REQUEST ) ; } }


Input: --------------------------------------------------------- Private Methods [CODESPLIT] private void listBeans ( JsonGenerator jg , ObjectName qry , String attribute , HttpServletResponse response ) throws IOException { Set < ObjectName > names = null ; names = mBeanServer . queryNames ( qry , null ) ; jg . writeArrayFieldStart ( "beans" ) ; Iterator < ObjectName > it = names . iterator ( ) ; while ( it . hasNext ( ) ) { ObjectName oname = it . next ( ) ; MBeanInfo minfo ; String code = "" ; Object attributeinfo = null ; try { minfo = mBeanServer . getMBeanInfo ( oname ) ; code = minfo . getClassName ( ) ; String prs = "" ; try { if ( "org.apache.commons.modeler.BaseModelMBean" . equals ( code ) ) { prs = "modelerType" ; code = ( String ) mBeanServer . getAttribute ( oname , prs ) ; } if ( attribute != null ) { prs = attribute ; attributeinfo = mBeanServer . getAttribute ( oname , prs ) ; } } catch ( AttributeNotFoundException e ) { // If the modelerType attribute was not found, the class name is used // instead. } catch ( MBeanException e ) { // The code inside the attribute getter threw an exception so // and fall back on the class name } catch ( RuntimeException e ) { // For some reason even with an MBeanException available to them // Runtime exceptionscan still find their way through, so treat them // the same as MBeanException } catch ( ReflectionException e ) { // This happens when the code inside the JMX bean (setter?? from the // java docs) threw an exception, so // class name } } catch ( InstanceNotFoundException e ) { //Ignored for some reason the bean was not found so don't output it continue ; } catch ( IntrospectionException e ) { // This is an internal error, something odd happened with reflection so // continue ; } catch ( ReflectionException e ) { // This happens when the code inside the JMX bean threw an exception, so // continue ; } jg . writeStartObject ( ) ; jg . writeStringField ( "name" , oname . toString ( ) ) ; jg . writeStringField ( "modelerType" , code ) ; if ( ( attribute != null ) && ( attributeinfo == null ) ) { jg . writeStringField ( "result" , "ERROR" ) ; jg . writeStringField ( "message" , "No attribute with name " + attribute + " was found." ) ; jg . writeEndObject ( ) ; jg . writeEndArray ( ) ; jg . close ( ) ; response . setStatus ( HttpServletResponse . SC_NOT_FOUND ) ; return ; } if ( attribute != null ) { writeAttribute ( jg , attribute , attributeinfo ) ; } else { MBeanAttributeInfo attrs [ ] = minfo . getAttributes ( ) ; for ( int i = 0 ; i < attrs . length ; i ++ ) { writeAttribute ( jg , oname , attrs [ i ] ) ; } } jg . writeEndObject ( ) ; } jg . writeEndArray ( ) ; }


Input: Refresh the schema for the table if the last update of this table was before the given SCN . Returns true if it was updated else returns false . [CODESPLIT] private boolean refreshSchema ( BigDecimal scnDecimal , SchemaAndTable schemaAndTable ) throws SQLException { try { if ( ! tableSchemaLastUpdate . containsKey ( schemaAndTable ) || scnDecimal . compareTo ( tableSchemaLastUpdate . get ( schemaAndTable ) ) > 0 ) { if ( containerized ) { try ( Statement switchToPdb = connection . createStatement ( ) ) { switchToPdb . execute ( "ALTER SESSION SET CONTAINER = " + configBean . pdb ) ; } } tableSchemas . put ( schemaAndTable , getTableSchema ( schemaAndTable ) ) ; tableSchemaLastUpdate . put ( schemaAndTable , scnDecimal ) ; return true ; } return false ; } finally { alterSession ( ) ; } }


Input: This method needs to get SQL like string with all required schemas and tables . [CODESPLIT] @ VisibleForTesting String getListOfSchemasAndTables ( List < SchemaAndTable > schemaAndTables ) { Map < String , List < String > > schemas = new HashMap <> ( ) ; for ( SchemaAndTable schemaAndTable : schemaAndTables ) { if ( schemas . containsKey ( schemaAndTable . getSchema ( ) ) ) { schemas . get ( schemaAndTable . getSchema ( ) ) . add ( schemaAndTable . getTable ( ) ) ; } else { List < String > tbls = new ArrayList <> ( ) ; tbls . add ( schemaAndTable . getTable ( ) ) ; schemas . put ( schemaAndTable . getSchema ( ) , tbls ) ; } } List < String > queries = new ArrayList <> ( ) ; for ( Map . Entry < String , List < String > > entry : schemas . entrySet ( ) ) { List < String > tables = new ArrayList <> ( ) ; int fromIndex = 0 ; int range = 1000 ; int maxIndex = entry . getValue ( ) . size ( ) ; int toIndex = range < maxIndex ? range : maxIndex ; while ( fromIndex < toIndex ) { tables . add ( Utils . format ( "TABLE_NAME IN ({})" , formatTableList ( entry . getValue ( ) . subList ( fromIndex , toIndex ) ) ) ) ; fromIndex = toIndex ; toIndex = ( toIndex + range ) < maxIndex ? ( toIndex + range ) : maxIndex ; } queries . add ( Utils . format ( "(SEG_OWNER='{}' AND ({}))" , entry . getKey ( ) , String . join ( " OR " , tables ) ) ) ; } return "( " + String . join ( " OR " , queries ) + " )" ; }


Input: An element is expired if the transaction started before the current window being processed and if no records have actually been sent to the pipeline . If a record has been sent then a commit was seen so it is not expired . [CODESPLIT] private boolean expired ( Map . Entry < TransactionIdKey , HashQueue < RecordSequence > > entry , LocalDateTime startTime ) { return startTime != null && // Can be null if starting from SCN and first batch is not complete yet. entry . getKey ( ) . txnStartTime . isBefore ( startTime . minusSeconds ( configBean . txnWindow ) ) && entry . getValue ( ) . peek ( ) . seq == 1 ; }


Input: offset will be negative if we are in truncate mode . [CODESPLIT] @ Override public long getOffset ( ) { Utils . checkState ( open , Utils . formatL ( "LiveFileReder for '{}' is not open" , currentFile ) ) ; return ( truncateMode ) ? - offset : offset ; }


Input: returns true if still in truncate mode false otherwise [CODESPLIT] private boolean fastForward ( ) throws IOException { try { boolean stillTruncate ; buffer . clear ( ) ; if ( channel . read ( buffer ) > - 1 || isEof ( ) ) { //set the buffer into read from mode buffer . flip ( ) ; //we have data, lets look for the first EOL in it. int firstEolIdx = findEndOfFirstLine ( buffer ) ; if ( firstEolIdx > - 1 ) { // set position to position after first EOL buffer . position ( firstEolIdx + 1 ) ; // set the buffer back into write into mode keeping data after first EOL buffer . compact ( ) ; stillTruncate = false ; offset = channel . position ( ) - buffer . position ( ) ; } else { // no EOL yet // whatever was read will be discarded on next next() call stillTruncate = true ; offset = channel . position ( ) ; } } else { // no data read // whatever was read will be discarded on next next() call stillTruncate = true ; offset = channel . position ( ) ; } return stillTruncate ; } catch ( IOException ex ) { closeChannel ( ) ; throw ex ; } }


Input: ----------------------------------------------- Private Members --------------------------------------------------- [CODESPLIT] private static void initRabbitConf ( Channel channel , BaseRabbitConfigBean conf ) throws IOException { // Channel is always bound to the default exchange. When specified, we must declare the exchange. for ( RabbitExchangeConfigBean exchange : conf . exchanges ) { channel . exchangeDeclare ( exchange . name , exchange . type . getValue ( ) , exchange . durable , exchange . autoDelete , exchange . declarationProperties ) ; } channel . queueDeclare ( conf . queue . name , conf . queue . durable , conf . queue . exclusive , conf . queue . autoDelete , conf . queue . properties ) ; for ( RabbitExchangeConfigBean exchange : conf . exchanges ) { bindQueue ( channel , conf , exchange ) ; } }


Input: Detached stage APIs [CODESPLIT] @ Path ( "/detachedstage" ) @ GET @ ApiOperation ( value = "Returns empty envelope for detached stage." , response = DetachedStageConfigurationJson . class , authorizations = @ Authorization ( value = "basic" ) ) @ Produces ( MediaType . APPLICATION_JSON ) @ RolesAllowed ( { AuthzRole . CREATOR , AuthzRole . ADMIN , AuthzRole . CREATOR_REMOTE , AuthzRole . ADMIN_REMOTE } ) public Response createDetachedStageEnvelope ( ) throws PipelineException { DetachedStageConfigurationJson detachedStage = new DetachedStageConfigurationJson ( new DetachedStageConfiguration ( ) ) ; return Response . ok ( ) . entity ( detachedStage ) . build ( ) ; }


Input: max array size will be limit + 1 ( because the magic byte is being added ) [CODESPLIT] static byte [ ] extract ( InputStream is , ByteArrayOutputStream overflowBuffer , int limit ) throws IOException { // the inputstream we get has been already stripped of the magic byte if first call byte [ ] message ; if ( copy ( is , overflowBuffer , limit - overflowBuffer . size ( ) ) ) { // got rest of payload without exceeding the max message size if ( overflowBuffer . size ( ) == 0 ) { // there is no more payload message = null ; } else { // extract the rest payload and prefix it with the magic byte byte [ ] data = overflowBuffer . toByteArray ( ) ; message = new byte [ data . length + 1 ] ; message [ 0 ] = JSON1_MAGIC_NUMBER ; System . arraycopy ( data , 0 , message , 1 , data . length ) ; overflowBuffer . reset ( ) ; } } else { // got partial payload, exceeded the max message size byte [ ] data = overflowBuffer . toByteArray ( ) ; // find last full record in partial payload int lastEOL = findEndOfLastLineBeforeLimit ( data , limit ) ; if ( lastEOL == - 1 ) { throw new IOException ( Utils . format ( "Maximum message size '{}' exceeded" , limit ) ) ; } // extract payload up to last EOL and prefix with the magic byte message = new byte [ lastEOL + 1 ] ; message [ 0 ] = JSON1_MAGIC_NUMBER ; System . arraycopy ( data , 0 , message , 1 , lastEOL ) ; // put back in the stream buffer the portion of the payload that did not make it to the message overflowBuffer . reset ( ) ; overflowBuffer . write ( data , lastEOL , data . length - lastEOL ) ; } return message ; }


Input: ContextExtensions [CODESPLIT] @ Override public RecordReader createRecordReader ( InputStream inputStream , long initialPosition , int maxObjectLen ) throws IOException { return RecordWriterReaderFactory . createRecordReader ( inputStream , initialPosition , maxObjectLen ) ; }


Input: We need to support Strings as some information that user might need to deal with is inherently stored in String variables - for example header values or CSV files . [CODESPLIT] private static Object convertStringToAppropriateNumber ( String value ) { if ( value . contains ( "." ) ) { return Double . valueOf ( value ) ; } else { return Long . valueOf ( value ) ; } }


Input: Parses a collectd packet part . [CODESPLIT] private int parsePart ( int startOffset , ByteBuf buf , Map < String , Field > fields ) throws OnRecordErrorException { int offset = startOffset ; int type = buf . getUnsignedShort ( offset ) ; // 0-1 offset += 2 ; final int length = buf . getUnsignedShort ( offset ) ; // 2-3 offset += 2 ; switch ( type ) { case HOST : case PLUGIN : case PLUGIN_INSTANCE : case TYPE : case TYPE_INSTANCE : case MESSAGE : pruneFields ( type ) ; fields . put ( PART_TYPES . get ( type ) , Field . create ( parseString ( offset , length , buf ) ) ) ; offset += length - 4 ; break ; case TIME_HIRES : case INTERVAL_HIRES : if ( type != INTERVAL_HIRES || ! excludeInterval ) { long value = parseNumeric ( offset , buf ) ; if ( convertTime ) { value *= ( Math . pow ( 2 , - 30 ) * 1000 ) ; type = type == TIME_HIRES ? TIME : INTERVAL ; } fields . put ( PART_TYPES . get ( type ) , Field . create ( value ) ) ; } offset += 8 ; break ; case TIME : case INTERVAL : case SEVERITY : if ( type != INTERVAL || ! excludeInterval ) { fields . put ( PART_TYPES . get ( type ) , Field . create ( parseNumeric ( offset , buf ) ) ) ; } offset += 8 ; break ; case VALUES : offset = parseValues ( offset , buf ) ; startNewRecord ( ) ; break ; case SIGNATURE : if ( ! verifySignature ( offset , length , buf ) ) { throw new OnRecordErrorException ( Errors . COLLECTD_02 ) ; } offset += length - 4 ; break ; case ENCRYPTION : String user = parseUser ( offset , buf ) ; offset += ( 2 + user . length ( ) ) ; byte [ ] iv = parseIv ( offset , buf ) ; offset += 16 ; decrypt ( offset , length , buf , user , iv ) ; // Skip the checksum and continue processing. offset += 20 ; break ; default : // Don't recognize this part type, so skip it LOG . warn ( "Unrecognized part type: {}" , type ) ; offset += length - 4 ; break ; } return offset ; }


Input: Parses the value part of the packet where metrics are located [CODESPLIT] private int parseValues ( int startOffset , ByteBuf buf ) throws OnRecordErrorException { int offset = startOffset ; // N Values // For each Value: // 1 byte data type code int numValues = buf . getUnsignedShort ( offset ) ; // 4-5 offset += 2 ; List < Byte > types = new ArrayList <> ( numValues ) ; while ( numValues -- > 0 ) { types . add ( buf . getByte ( offset ) ) ; offset += 1 ; } for ( int i = 0 ; i < types . size ( ) ; i ++ ) { Byte type = types . get ( i ) ; String label = getValueLabel ( i , type ) ; switch ( type ) { case COUNTER : fields . put ( label , Field . create ( buf . getUnsignedInt ( offset ) ) ) ; offset += 8 ; break ; case GAUGE : fields . put ( label , Field . create ( buf . order ( ByteOrder . LITTLE_ENDIAN ) . getDouble ( offset ) ) ) ; offset += 8 ; break ; case DERIVE : fields . put ( label , Field . create ( buf . getLong ( offset ) ) ) ; offset += 8 ; break ; case ABSOLUTE : fields . put ( label , Field . create ( buf . getUnsignedInt ( offset ) ) ) ; offset += 8 ; break ; default : // error throw new OnRecordErrorException ( Errors . COLLECTD_01 , type ) ; } } return offset ; }


Input: Adapted from https : // stackoverflow . com / a / 7932774 / 375670 [CODESPLIT] public static Field getUnsignedShortField ( byte [ ] bytes ) { Utils . checkState ( bytes . length == 2 , "2 bytes required to parse an unsigned short" ) ; final short shortVal = Shorts . fromByteArray ( bytes ) ; int intVal = shortVal >= 0 ? shortVal : 0x10000 + shortVal ; return Field . create ( intVal ) ; }


Input: It takes a record structure in <String HiveTypeInfo > format . Generate a schema and return in String . [CODESPLIT] @ Override public String inferSchema ( Map < String , HiveTypeInfo > record ) throws StageException { Map < String , Schema > fields = new LinkedHashMap <> ( ) ; for ( Map . Entry < String , HiveTypeInfo > pair : record . entrySet ( ) ) { if ( ! HiveMetastoreUtil . validateObjectName ( pair . getKey ( ) ) ) { throw new HiveStageCheckedException ( Errors . HIVE_30 , pair . getKey ( ) ) ; } Schema columnSchema = Schema . createUnion ( ImmutableList . of ( Schema . create ( Schema . Type . NULL ) , traverse ( pair ) ) ) ; // We always set default value to null columnSchema . addProp ( "default" , NullNode . getInstance ( ) ) ; fields . put ( pair . getKey ( ) , columnSchema ) ; } Schema schema = buildSchema ( fields ) ; return schema . toString ( ) ; }


Input: Our own implementation of JDBCType . valueOf () that won t throw an exception in case of unknown type . [CODESPLIT] public static String nameForType ( DatabaseVendor vendor , int jdbcType ) { for ( JDBCType sqlType : JDBCType . class . getEnumConstants ( ) ) { if ( jdbcType == sqlType . getVendorTypeNumber ( ) ) return sqlType . name ( ) ; } switch ( vendor ) { case ORACLE : switch ( jdbcType ) { case - 101 : return "TIMESTAMP WITH TIME ZONE" ; case - 102 : return "TIMESTAMP WITH LOCAL TIME ZONE" ; } break ; } return "Unknown" ; }


Input: Returns qualified table name ( schema . table name ) [CODESPLIT] public static String getQualifiedTableName ( String schema , String tableName ) { return StringUtils . isEmpty ( schema ) ? tableName : schema + "." + tableName ; }


Input: Returns quoted qualified table name ( schema . table name ) based on quoteChar [CODESPLIT] public static String getQuotedQualifiedTableName ( String schema , String tableName , String qC ) { String quotedTableName = String . format ( OffsetQueryUtil . QUOTED_NAME , qC , tableName , qC ) ; return StringUtils . isEmpty ( schema ) ? quotedTableName : String . format ( OffsetQueryUtil . QUOTED_NAME , qC , schema , qC ) + "." + quotedTableName ; }


Input: Evaluate ELs in Initial offsets as needed and populate the final String representation of initial offsets in { [CODESPLIT] private void populateInitialOffset ( PushSource . Context context , List < Stage . ConfigIssue > issues , Map < String , String > configuredColumnToInitialOffset , TableJdbcELEvalContext tableJdbcELEvalContext , Map < String , String > offsetColumnToStartOffset ) throws StageException { for ( Map . Entry < String , String > partitionColumnInitialOffsetEntry : configuredColumnToInitialOffset . entrySet ( ) ) { String value ; try { value = tableJdbcELEvalContext . evaluateAsString ( "offsetColumnToInitialOffsetValue" , partitionColumnInitialOffsetEntry . getValue ( ) ) ; if ( value == null ) { issues . add ( context . createConfigIssue ( Groups . TABLE . name ( ) , TableJdbcConfigBean . TABLE_CONFIG , JdbcErrors . JDBC_73 , partitionColumnInitialOffsetEntry . getValue ( ) , Utils . format ( "Expression returned date as null. Check Expression" ) ) ) ; return ; } } catch ( ELEvalException e ) { issues . add ( context . createConfigIssue ( Groups . TABLE . name ( ) , TableJdbcConfigBean . TABLE_CONFIG , JdbcErrors . JDBC_73 , partitionColumnInitialOffsetEntry . getValue ( ) , e ) ) ; return ; } offsetColumnToStartOffset . put ( partitionColumnInitialOffsetEntry . getKey ( ) , value ) ; } }


Input: Lists all tables matching the { [CODESPLIT] public Map < String , TableContext > listTablesForConfig ( DatabaseVendor vendor , PushSource . Context context , List < Stage . ConfigIssue > issues , Connection connection , TableConfigBean tableConfigBean , TableJdbcELEvalContext tableJdbcELEvalContext , QuoteChar quoteChar ) throws SQLException , StageException { Map < String , TableContext > tableContextMap = new LinkedHashMap <> ( ) ; Pattern tableExclusion = StringUtils . isEmpty ( tableConfigBean . tableExclusionPattern ) ? null : Pattern . compile ( tableConfigBean . tableExclusionPattern ) ; Pattern schemaExclusion = StringUtils . isEmpty ( tableConfigBean . schemaExclusionPattern ) ? null : Pattern . compile ( tableConfigBean . schemaExclusionPattern ) ; try ( ResultSet rs = jdbcUtil . getTableAndViewMetadata ( connection , tableConfigBean . schema , tableConfigBean . tablePattern ) ) { while ( rs . next ( ) ) { String schemaName = rs . getString ( TABLE_METADATA_TABLE_SCHEMA_CONSTANT ) ; String tableName = rs . getString ( TABLE_METADATA_TABLE_NAME_CONSTANT ) ; if ( ( tableExclusion == null || ! tableExclusion . matcher ( tableName ) . matches ( ) ) && ( schemaExclusion == null || ! schemaExclusion . matcher ( schemaName ) . matches ( ) ) ) { TableContext tableContext = createTableContext ( vendor , context , issues , connection , schemaName , tableName , tableConfigBean , tableJdbcELEvalContext , quoteChar ) ; if ( tableContext != null ) { tableContextMap . put ( getQualifiedTableName ( schemaName , tableName ) , tableContext ) ; } } } } return tableContextMap ; }


Input: Access the database obtain a list of primary key columns and store them in primaryKeyColumns . If table has no primary keys primaryKeyColumns stays empty . [CODESPLIT] void lookupPrimaryKeys ( ) throws StageException { Connection connection = null ; try { connection = dataSource . getConnection ( ) ; primaryKeyColumns = jdbcUtil . getPrimaryKeys ( connection , schema , tableName ) ; } catch ( SQLException e ) { String formattedError = jdbcUtil . formatSqlException ( e ) ; LOG . error ( formattedError , e ) ; throw new StageException ( JdbcErrors . JDBC_17 , tableName , formattedError ) ; } finally { if ( connection != null ) { try { connection . close ( ) ; } catch ( SQLException e ) { String formattedError = jdbcUtil . formatSqlException ( e ) ; LOG . error ( formattedError , e ) ; } } } }


Input: Access database and obtain the metadata for the table . Store columnName and / columnName to the columnsToFields map as a default column - to - field mapping . Store columnName and ? to columnsToParameters map as a default column - to - value mapping . They will be updated later in createCustomFieldMappings () . [CODESPLIT] private void createDefaultFieldMappings ( ) throws StageException { try ( Connection connection = dataSource . getConnection ( ) ) { try ( ResultSet res = jdbcUtil . getTableMetadata ( connection , schema , tableName ) ) { if ( ! res . next ( ) ) { throw new StageException ( JdbcErrors . JDBC_16 , getTableName ( ) ) ; } } try ( ResultSet columns = jdbcUtil . getColumnMetadata ( connection , schema , tableName ) ) { while ( columns . next ( ) ) { String columnName = columns . getString ( COLUMN_NAME ) ; columnsToFields . put ( columnName , "/" + columnName ) ; // Default implicit field mappings columnsToParameters . put ( columnName , "?" ) ; columnType . put ( columnName , columns . getInt ( DATA_TYPE ) ) ; } } } catch ( SQLException e ) { String errorMessage = jdbcUtil . formatSqlException ( e ) ; LOG . error ( errorMessage ) ; LOG . debug ( errorMessage , e ) ; throw new StageException ( JdbcErrors . JDBC_09 , tableName ) ; } }


Input: Use Field to Column Mapping option obtained from configuration and update columnsToFields and columnsToParameters . [CODESPLIT] private void createCustomFieldMappings ( ) { for ( JdbcFieldColumnParamMapping mapping : customMappings ) { LOG . debug ( "Custom mapping field {} to column {}" , mapping . field , mapping . columnName ) ; if ( columnsToFields . containsKey ( mapping . columnName ) ) { LOG . debug ( "Mapping field {} to column {}" , mapping . field , mapping . columnName ) ; columnsToFields . put ( mapping . columnName , mapping . field ) ; columnsToParameters . put ( mapping . columnName , mapping . paramValue ) ; } } }


Input: spec requires a string name for a data type rather than just an enum . [CODESPLIT] static String getSQLTypeName ( Field . Type type ) throws OnRecordErrorException { switch ( type ) { case BOOLEAN : return "BOOLEAN" ; case CHAR : return "CHAR" ; case BYTE : return "BINARY" ; case SHORT : return "SMALLINT" ; case INTEGER : return "INTEGER" ; case LONG : return "BIGINT" ; case FLOAT : return "FLOAT" ; case DOUBLE : return "DOUBLE" ; case DATE : return "DATE" ; case TIME : return "TIME" ; case DATETIME : return "TIMESTAMP" ; case DECIMAL : return "DECIMAL" ; case STRING : return "VARCHAR" ; case BYTE_ARRAY : return "VARBINARY" ; case LIST_MAP : case MAP : throw new OnRecordErrorException ( JdbcErrors . JDBC_05 , "Unsupported list or map type: MAP" ) ; case LIST : return "ARRAY" ; default : throw new OnRecordErrorException ( JdbcErrors . JDBC_05 , "Unsupported type: " + type . name ( ) ) ; } }


Input: Table this writer will write to . [CODESPLIT] protected String getTableName ( ) { if ( ! Strings . isNullOrEmpty ( schema ) ) { if ( caseSensitive ) { return "\"" + schema + "\"." + "\"" + tableName + "\"" ; } else { return schema + "." + tableName ; } } if ( caseSensitive ) { return "\"" + tableName + "\"" ; } return tableName ; }


Input: Set primary key values to query . This is called only for UPDATE and DELETE operations . If primary key value is missing in record it throws OnRecordErrorException . [CODESPLIT] int setPrimaryKeys ( int index , final Record record , PreparedStatement statement , int opCode ) throws OnRecordErrorException { for ( String key : getPrimaryKeyColumns ( ) ) { Field field = record . get ( recordReader . getFieldPath ( key , getColumnsToFields ( ) , opCode ) ) ; if ( field == null ) { LOG . error ( "Primary key {} is missing in record" , key ) ; throw new OnRecordErrorException ( record , JdbcErrors . JDBC_19 , key ) ; } Object value = field . getValue ( ) ; try { statement . setObject ( index , value , getColumnType ( key ) ) ; } catch ( SQLException ex ) { LOG . error ( "SQLException thrown: {}" , ex . getMessage ( ) ) ; throw new OnRecordErrorException ( record , JdbcErrors . JDBC_19 , key , ex ) ; } ++ index ; } return index ; }


Input: This is an error that is not due to bad input record and should throw a StageException once we format the error . [CODESPLIT] void handleSqlException ( SQLException e ) throws StageException { String formattedError = jdbcUtil . formatSqlException ( e ) ; LOG . error ( formattedError , e ) ; throw new StageException ( JdbcErrors . JDBC_14 , e . getSQLState ( ) , e . getErrorCode ( ) , e . getMessage ( ) , formattedError , e ) ; }


Input: Get the numeric operation code from record header . The default code is used if the operation code is not found in the header . [CODESPLIT] protected int getOperationCode ( Record record , List < OnRecordErrorException > errorRecords ) { return recordReader . getOperationFromRecord ( record , defaultOpCode , unsupportedAction , errorRecords ) ; }


Input: Split records according to tuples ( schemaName tableName ) [CODESPLIT] public Multimap < SchemaAndTable , Record > classify ( Batch batch ) throws OnRecordErrorException { Multimap < SchemaAndTable , Record > partitions = ArrayListMultimap . create ( ) ; Iterator < Record > batchIterator = batch . getRecords ( ) ; while ( batchIterator . hasNext ( ) ) { Record record = batchIterator . next ( ) ; String schemaName = schemaNameExpr ; String tableName = tableNameExpr ; if ( dynamicSchemaName ) { try { RecordEL . setRecordInContext ( schemaNameVars , record ) ; schemaName = schemaNameEval . eval ( schemaNameVars , schemaNameExpr , String . class ) ; LOG . debug ( "Expression '{}' is evaluated to '{}' : " , schemaNameExpr , schemaName ) ; } catch ( ELEvalException e ) { LOG . error ( "Failed to evaluate expression '{}' : " , schemaNameExpr , e . toString ( ) , e ) ; throw new OnRecordErrorException ( record , e . getErrorCode ( ) , e . getParams ( ) ) ; } } if ( dynamicTableName ) { try { RecordEL . setRecordInContext ( tableNameVars , record ) ; tableName = tableNameEval . eval ( tableNameVars , tableNameExpr , String . class ) ; LOG . debug ( "Expression '{}' is evaluated to '{}' : " , tableNameExpr , tableName ) ; } catch ( ELEvalException e ) { LOG . error ( "Failed to evaluate expression '{}' : " , tableNameExpr , e . toString ( ) , e ) ; throw new OnRecordErrorException ( record , e . getErrorCode ( ) , e . getParams ( ) ) ; } } SchemaAndTable partitionKey = new SchemaAndTable ( schemaName , tableName ) ; partitions . put ( partitionKey , record ) ; } return partitions ; }


Input: Process method for Push source that will give control of the execution to the origin . [CODESPLIT] public void process ( Map < String , String > offsets , int batchSize , ReportErrorDelegate reportErrorDelegate ) throws StageException , PipelineRuntimeException { this . reportErrorDelegate = reportErrorDelegate ; getStage ( ) . setReportErrorDelegate ( this ) ; try { MDC . put ( LogConstants . STAGE , getStage ( ) . getInfo ( ) . getInstanceName ( ) ) ; getStage ( ) . execute ( offsets , batchSize ) ; } finally { MDC . put ( LogConstants . STAGE , "" ) ; } }


Input: Called by PipelineRunner when push origin started a new batch to prepare context for it . [CODESPLIT] public void prepareBatchContext ( BatchContextImpl batchContext ) { PipeBatch pipeBatch = batchContext . getPipeBatch ( ) ; // Start stage in the pipe batch and persist reference to batch maker in the batch context BatchMakerImpl batchMaker = pipeBatch . startStage ( this ) ; batchContext . setBatchMaker ( batchMaker ) ; batchContext . setOriginStageName ( getStage ( ) . getInfo ( ) . getInstanceName ( ) , getStage ( ) . getInfo ( ) . getLabel ( ) ) ; }


Input: Finish batch from the origin s perspective . [CODESPLIT] public Map < String , Object > finishBatchContext ( BatchContextImpl batchContext ) throws StageException { return finishBatchAndCalculateMetrics ( batchContext . getStartTime ( ) , batchContext . getPipeBatch ( ) , ( BatchMakerImpl ) batchContext . getBatchMaker ( ) , batchContext . getPipeBatch ( ) . getBatch ( this ) , batchContext . getPipeBatch ( ) . getErrorSink ( ) , batchContext . getPipeBatch ( ) . getEventSink ( ) , null ) ; }


Input: Flatten the entire record to one giant map [CODESPLIT] private Map < String , Field > flattenEntireRecord ( Field rootField ) { Map < String , Field > ret = new LinkedHashMap <> ( ) ; switch ( rootField . getType ( ) ) { case MAP : case LIST_MAP : flattenMap ( "" , rootField . getValueAsMap ( ) , ret ) ; break ; case LIST : flattenList ( "" , rootField . getValueAsList ( ) , ret ) ; break ; default : break ; } return ret ; }


Input: Create new instance of detached stage runtime . [CODESPLIT] public static < T > DetachedStageRuntime < ? extends T > create ( StageBean bean , Stage . Info info , Stage . Context context , Class < T > klass ) { switch ( bean . getDefinition ( ) . getType ( ) ) { case PROCESSOR : return new DetachedStageRuntime . DetachedProcessor ( bean , info , context ) ; case TARGET : case EXECUTOR : return new DetachedStageRuntime . DetachedTarget ( bean , info , context ) ; default : throw new RuntimeException ( "Unsupported stage type: " + bean . getDefinition ( ) . getType ( ) ) ; } }


Input: Executes an action for given record . [CODESPLIT] private void execute ( Record record ) throws OnRecordErrorException { // This is a contrived example, normally you may be performing an operation that could throw // an exception or produce an error condition. In that case you can throw an OnRecordErrorException // to send this record to the error pipeline with some details. if ( ! record . has ( "/someField" ) ) { throw new OnRecordErrorException ( Errors . SAMPLE_01 , record , "exception detail message." ) ; } // TODO: execute action }


Input: Records from MySQL BinLog origin have a bit unique structure . [CODESPLIT] @ Override public SortedMap < String , String > getColumnsToParameters ( final Record record , int op , Map < String , String > parameters , Map < String , String > columnsToFields ) { SortedMap < String , String > columnsToParameters = new TreeMap <> ( ) ; for ( Map . Entry < String , String > entry : columnsToFields . entrySet ( ) ) { String columnName = entry . getKey ( ) ; String fieldPath = entry . getValue ( ) ; if ( op == OperationType . DELETE_CODE ) { fieldPath = fieldPath . replace ( DATA_FIELD , OLD_DATA_FIELD ) ; } if ( record . has ( fieldPath ) ) { columnsToParameters . put ( columnName , parameters . get ( columnName ) ) ; } } return columnsToParameters ; }


Input: Return a fieldpath for the corresponding column name . The columnToField Map contains column - to - field mapping so simply get fieldpath for the column name if operation is INSERT or UPDATE . We replace / Data to / OldData for delete operation because that s where records store primary key info . [CODESPLIT] @ Override String getFieldPath ( String columnName , Map < String , String > columnsToField , int op ) { if ( op == OperationType . DELETE_CODE ) { String fieldPath = columnsToField . get ( columnName ) ; if ( fieldPath == null ) { LOG . error ( "Column name {} is not defined in column-filed mapping" , columnName ) ; return null ; } return fieldPath . replace ( "/Data" , "/OldData" ) ; } // For insert and update, ok to use the field name set by column-field mapping return columnsToField . get ( columnName ) ; }


Input: Build query using the lastOffset which is of the form ( <column1 > = <value1 > :: <column2 > = <value2 > :: <column3 > = <value3 > ) [CODESPLIT] public static Pair < String , List < Pair < Integer , String > > > buildAndReturnQueryAndParamValToSet ( TableRuntimeContext tableRuntimeContext , String lastOffset , String quoteChar , TableJdbcELEvalContext tableJdbcELEvalContext ) throws ELEvalException { final TableContext tableContext = tableRuntimeContext . getSourceTableContext ( ) ; StringBuilder queryBuilder = new StringBuilder ( ) ; List < Pair < Integer , String > > paramValueToSet = new ArrayList <> ( ) ; queryBuilder . append ( buildBaseTableQuery ( tableRuntimeContext , quoteChar ) ) ; Map < String , String > storedTableToOffset = getColumnsToOffsetMapFromOffsetFormat ( lastOffset ) ; final boolean noStoredOffsets = storedTableToOffset . isEmpty ( ) ; //Determines whether an initial offset is specified in the config and there is no stored offset. boolean isOffsetOverriden = tableContext . isOffsetOverriden ( ) && noStoredOffsets ; OffsetComparison minComparison = OffsetComparison . GREATER_THAN ; Map < String , String > offset = null ; if ( isOffsetOverriden ) { //Use the offset in the configuration offset = tableContext . getOffsetColumnToStartOffset ( ) ; } else if ( tableRuntimeContext . isPartitioned ( ) && noStoredOffsets ) { // use partitioned starting offsets offset = tableRuntimeContext . getStartingPartitionOffsets ( ) ; minComparison = OffsetComparison . GREATER_THAN_OR_EQUALS ; } else { // if offset is available // get the stored offset (which is of the form partitionName=value) and strip off 'offsetColumns=' prefix // else null // offset = storedOffsets; offset = storedTableToOffset ; } Map < String , String > maxOffsets = new HashMap <> ( ) ; if ( tableRuntimeContext . isPartitioned ( ) ) { maxOffsets . putAll ( tableRuntimeContext . getMaxPartitionOffsets ( ) ) ; } List < String > finalAndConditions = new ArrayList <> ( ) ; //Apply last offset conditions if ( offset != null && ! offset . isEmpty ( ) ) { List < String > finalOrConditions = new ArrayList <> ( ) ; List < String > preconditions = new ArrayList <> ( ) ; List < Pair < Integer , String > > preconditionParamVals = new ArrayList <> ( ) ; //For partition columns p1, p2 and p3 with offsets o1, o2 and o3 respectively, the query will look something like //select * from tableName where (p1 > o1) or (p1 = o1 and p2 > o2) or (p1 = o1 and p2 = o2 and p3 > o3) order by p1, p2, p3. for ( String partitionColumn : tableContext . getOffsetColumns ( ) ) { int partitionSqlType = tableContext . getOffsetColumnType ( partitionColumn ) ; String partitionOffset = offset . get ( partitionColumn ) ; String thisPartitionColumnMax = null ; boolean hasMaxOffset = false ; // add max value for partition column (if applicable) if ( maxOffsets . containsKey ( partitionColumn ) ) { final String maxOffset = maxOffsets . get ( partitionColumn ) ; if ( ! Strings . isNullOrEmpty ( maxOffset ) ) { Pair < Integer , String > paramValForCurrentOffsetColumnMax = Pair . of ( partitionSqlType , maxOffset ) ; // add for current partition column max value paramValueToSet . add ( paramValForCurrentOffsetColumnMax ) ; thisPartitionColumnMax = getConditionForPartitionColumn ( partitionColumn , OffsetComparison . LESS_THAN , preconditions , quoteChar ) ; hasMaxOffset = true ; } } final String thisPartitionColumnMin = getConditionForPartitionColumn ( partitionColumn , minComparison , preconditions , quoteChar ) ; String conditionForThisPartitionColumn ; if ( hasMaxOffset ) { conditionForThisPartitionColumn = String . format ( AND_CONDITION_FORMAT , // add max condition first, since its param to set was already added thisPartitionColumnMax , thisPartitionColumnMin ) ; } else { conditionForThisPartitionColumn = String . format ( CONDITION_FORMAT , thisPartitionColumnMin ) ; } //Add for preconditions (EX: composite keys) paramValueToSet . addAll ( new ArrayList <> ( preconditionParamVals ) ) ; Pair < Integer , String > paramValForCurrentOffsetColumn = Pair . of ( partitionSqlType , partitionOffset ) ; //Add for current partition column paramValueToSet . add ( paramValForCurrentOffsetColumn ) ; finalOrConditions . add ( conditionForThisPartitionColumn ) ; preconditions . add ( getConditionForPartitionColumn ( partitionColumn , OffsetComparison . EQUALS , Collections . emptyList ( ) , quoteChar ) ) ; preconditionParamVals . add ( paramValForCurrentOffsetColumn ) ; } finalAndConditions . add ( String . format ( CONDITION_FORMAT , OR_JOINER . join ( finalOrConditions ) ) ) ; } if ( ! StringUtils . isEmpty ( tableContext . getExtraOffsetColumnConditions ( ) ) ) { //Apply extra offset column conditions configured which will be appended as AND on the query String condition = tableJdbcELEvalContext . evaluateAsString ( "extraOffsetColumnConditions" , tableContext . getExtraOffsetColumnConditions ( ) ) ; finalAndConditions . add ( String . format ( CONDITION_FORMAT , condition ) ) ; } if ( ! finalAndConditions . isEmpty ( ) ) { queryBuilder . append ( String . format ( WHERE_CLAUSE , AND_JOINER . join ( finalAndConditions ) ) ) ; } Collection < String > quotedOffsetColumns = tableContext . getOffsetColumns ( ) . stream ( ) . map ( offsetCol -> String . format ( QUOTED_NAME , quoteChar , offsetCol , quoteChar ) ) . collect ( Collectors . toList ( ) ) ; queryBuilder . append ( String . format ( ORDER_BY_CLAUSE , COMMA_SPACE_JOINER . join ( quotedOffsetColumns ) ) ) ; return Pair . of ( queryBuilder . toString ( ) , paramValueToSet ) ; }


Input: Builds parts of the query in the where clause for the the partitition column . [CODESPLIT] private static String getConditionForPartitionColumn ( String partitionColumn , OffsetComparison comparison , List < String > preconditions , String quoteChar ) { String conditionTemplate = comparison . getQueryCondition ( ) ; List < String > finalConditions = new ArrayList <> ( preconditions ) ; finalConditions . add ( String . format ( conditionTemplate , String . format ( QUOTED_NAME , quoteChar , partitionColumn , quoteChar ) , PREPARED_STATEMENT_POSITIONAL_PARAMETER ) ) ; return AND_JOINER . join ( finalConditions ) ; }


Input: Splits the offset in the form of ( <column1 > = <value1 > :: <column2 > = <value2 > :: <column3 > = <value3 > ) into a map of columns and values [CODESPLIT] public static Map < String , String > getColumnsToOffsetMapFromOffsetFormat ( String lastOffset ) { Map < String , String > offsetColumnsToOffsetMap = new HashMap <> ( ) ; if ( StringUtils . isNotBlank ( lastOffset ) ) { Iterator < String > offsetColumnsAndOffsetIterator = OFFSET_COLUMN_SPLITTER . split ( lastOffset ) . iterator ( ) ; while ( offsetColumnsAndOffsetIterator . hasNext ( ) ) { String offsetColumnAndOffset = offsetColumnsAndOffsetIterator . next ( ) ; String [ ] offsetColumnOffsetSplit = offsetColumnAndOffset . split ( "=" ) ; String offsetColumn = offsetColumnOffsetSplit [ 0 ] ; String offset = offsetColumnOffsetSplit [ 1 ] ; offsetColumnsToOffsetMap . put ( offsetColumn , offset ) ; } } return offsetColumnsToOffsetMap ; }


Input: Joins the map of column to values to a string offset in the form of ( <column1 > = <value1 > :: <column2 > = <value2 > :: <column3 > = <value3 > ) [CODESPLIT] public static String getOffsetFormatFromColumns ( TableRuntimeContext tableContext , Map < String , Field > fields ) throws StageException { return getOffsetFormat ( getOffsetsFromColumns ( tableContext , fields ) ) ; }


Input: This is the inverse of { @link #getSourceKeyOffsetsRepresentation ( Map ) } . [CODESPLIT] public static Map < String , String > getOffsetsFromSourceKeyRepresentation ( String offsets ) { final Map < String , String > offsetMap = new HashMap <> ( ) ; if ( StringUtils . isNotBlank ( offsets ) ) { for ( String col : StringUtils . splitByWholeSeparator ( offsets , OFFSET_KEY_COLUMN_SEPARATOR ) ) { final String [ ] parts = StringUtils . splitByWholeSeparator ( col , OFFSET_KEY_COLUMN_NAME_VALUE_SEPARATOR , 2 ) ; if ( parts . length != 2 ) { throw new IllegalArgumentException ( String . format ( "Invalid column offset of \"%s\" seen.  Expected colName%svalue.  Full offsets representation: %s" , col , OFFSET_KEY_COLUMN_NAME_VALUE_SEPARATOR , offsets ) ) ; } offsetMap . put ( parts [ 0 ] , parts [ 1 ] ) ; } } return offsetMap ; }


Input: Validates whether offset names match in the stored offset with respect to table configuration [CODESPLIT] public static Map < String , String > validateStoredAndSpecifiedOffset ( TableContext tableContext , String offset ) throws StageException { Set < String > expectedColumns = Sets . newHashSet ( tableContext . getOffsetColumns ( ) ) ; final Map < String , String > actualOffsets = getColumnsToOffsetMapFromOffsetFormat ( offset ) ; // only perform the actual validation below if there ARE stored offsets if ( actualOffsets . size ( ) == 0 ) { return actualOffsets ; } Set < String > actualColumns = actualOffsets . keySet ( ) ; Set < String > expectedSetDifference = Sets . difference ( expectedColumns , actualColumns ) ; Set < String > actualSetDifference = Sets . difference ( actualColumns , expectedColumns ) ; if ( expectedSetDifference . size ( ) > 0 || actualSetDifference . size ( ) > 0 ) { throw new StageException ( JdbcErrors . JDBC_71 , tableContext . getQualifiedName ( ) , COMMA_SPACE_JOINER . join ( actualColumns ) , COMMA_SPACE_JOINER . join ( expectedColumns ) ) ; } return actualOffsets ; }


Input: Return InputStream from which a new generated resource bundle can be retrieved . [CODESPLIT] public SupportBundle generateNewBundle ( List < String > generators , BundleType bundleType ) throws IOException { List < BundleContentGeneratorDefinition > defs = getRequestedDefinitions ( generators ) ; return generateNewBundleFromInstances ( defs . stream ( ) . map ( BundleContentGeneratorDefinition :: createInstance ) . collect ( Collectors . toList ( ) ) , bundleType ) ; }


Input: Return InputStream from which a new generated resource bundle can be retrieved . [CODESPLIT] public SupportBundle generateNewBundleFromInstances ( List < BundleContentGenerator > generators , BundleType bundleType ) throws IOException { PipedInputStream inputStream = new PipedInputStream ( ) ; PipedOutputStream outputStream = new PipedOutputStream ( ) ; inputStream . connect ( outputStream ) ; ZipOutputStream zipOutputStream = new ZipOutputStream ( outputStream ) ; executor . submit ( ( ) -> generateNewBundleInternal ( generators , bundleType , zipOutputStream ) ) ; String bundleName = generateBundleName ( bundleType ) ; String bundleKey = generateBundleDate ( bundleType ) + "/" + bundleName ; return new SupportBundle ( bundleKey , bundleName , inputStream ) ; }


Input: Instead of providing support bundle directly to user upload it to StreamSets backend services . [CODESPLIT] public void uploadNewBundle ( List < String > generators , BundleType bundleType ) throws IOException { List < BundleContentGeneratorDefinition > defs = getRequestedDefinitions ( generators ) ; uploadNewBundleFromInstances ( defs . stream ( ) . map ( BundleContentGeneratorDefinition :: createInstance ) . collect ( Collectors . toList ( ) ) , bundleType ) ; }


Input: Instead of providing support bundle directly to user upload it to StreamSets backend services . [CODESPLIT] public void uploadNewBundleFromInstances ( List < BundleContentGenerator > generators , BundleType bundleType ) throws IOException { // Generate bundle SupportBundle bundle = generateNewBundleFromInstances ( generators , bundleType ) ; boolean enabled = configuration . get ( Constants . UPLOAD_ENABLED , Constants . DEFAULT_UPLOAD_ENABLED ) ; String accessKey = configuration . get ( Constants . UPLOAD_ACCESS , Constants . DEFAULT_UPLOAD_ACCESS ) ; String secretKey = configuration . get ( Constants . UPLOAD_SECRET , Constants . DEFAULT_UPLOAD_SECRET ) ; String bucket = configuration . get ( Constants . UPLOAD_BUCKET , Constants . DEFAULT_UPLOAD_BUCKET ) ; int bufferSize = configuration . get ( Constants . UPLOAD_BUFFER_SIZE , Constants . DEFAULT_UPLOAD_BUFFER_SIZE ) ; if ( ! enabled ) { throw new IOException ( "Uploading support bundles was disabled by administrator." ) ; } AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider ( new BasicAWSCredentials ( accessKey , secretKey ) ) ; AmazonS3Client s3Client = new AmazonS3Client ( credentialsProvider , new ClientConfiguration ( ) ) ; s3Client . setS3ClientOptions ( new S3ClientOptions ( ) . withPathStyleAccess ( true ) ) ; s3Client . setRegion ( Region . getRegion ( Regions . US_WEST_2 ) ) ; // Object Metadata ObjectMetadata s3Metadata = new ObjectMetadata ( ) ; for ( Map . Entry < Object , Object > entry : getMetadata ( bundleType ) . entrySet ( ) ) { s3Metadata . addUserMetadata ( ( String ) entry . getKey ( ) , ( String ) entry . getValue ( ) ) ; } List < PartETag > partETags ; InitiateMultipartUploadResult initResponse = null ; try { // Uploading part by part LOG . info ( "Initiating multi-part support bundle upload" ) ; partETags = new ArrayList <> ( ) ; InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest ( bucket , bundle . getBundleKey ( ) ) ; initRequest . setObjectMetadata ( s3Metadata ) ; initResponse = s3Client . initiateMultipartUpload ( initRequest ) ; } catch ( AmazonClientException e ) { LOG . error ( "Support bundle upload failed: " , e ) ; throw new IOException ( "Support bundle upload failed" , e ) ; } try { byte [ ] buffer = new byte [ bufferSize ] ; int partId = 1 ; int size = - 1 ; while ( ( size = readFully ( bundle . getInputStream ( ) , buffer ) ) != - 1 ) { LOG . debug ( "Uploading part {} of size {}" , partId , size ) ; UploadPartRequest uploadRequest = new UploadPartRequest ( ) . withBucketName ( bucket ) . withKey ( bundle . getBundleKey ( ) ) . withUploadId ( initResponse . getUploadId ( ) ) . withPartNumber ( partId ++ ) . withInputStream ( new ByteArrayInputStream ( buffer ) ) . withPartSize ( size ) ; partETags . add ( s3Client . uploadPart ( uploadRequest ) . getPartETag ( ) ) ; } CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest ( bucket , bundle . getBundleKey ( ) , initResponse . getUploadId ( ) , partETags ) ; s3Client . completeMultipartUpload ( compRequest ) ; LOG . info ( "Support bundle upload finished" ) ; } catch ( Exception e ) { LOG . error ( "Support bundle upload failed" , e ) ; s3Client . abortMultipartUpload ( new AbortMultipartUploadRequest ( bucket , bundle . getBundleKey ( ) , initResponse . getUploadId ( ) ) ) ; throw new IOException ( "Can't upload support bundle" , e ) ; } finally { // Close the client s3Client . shutdown ( ) ; } }


Input: Try to upload bundle as part of internal SDC error ( for example failing pipeline ) . [CODESPLIT] public void uploadNewBundleOnError ( ) { boolean enabled = configuration . get ( Constants . UPLOAD_ON_ERROR , Constants . DEFAULT_UPLOAD_ON_ERROR ) ; LOG . info ( "Upload bundle on error: {}" , enabled ) ; // We won't upload the bundle unless it's explicitly allowed if ( ! enabled ) { return ; } try { uploadNewBundle ( Collections . emptyList ( ) , BundleType . SUPPORT ) ; } catch ( IOException e ) { LOG . error ( "Failed to upload error bundle" , e ) ; } }


Input: This method will read from the input stream until the whole buffer is loaded up with actual bytes or end of stream has been reached . Hence it will return buffer . length of all executions except the last two - one to the last call will return less then buffer . length ( reminder of the data ) and returns - 1 on any subsequent calls . [CODESPLIT] private int readFully ( InputStream inputStream , byte [ ] buffer ) throws IOException { int readBytes = 0 ; while ( readBytes < buffer . length ) { int loaded = inputStream . read ( buffer , readBytes , buffer . length - readBytes ) ; if ( loaded == - 1 ) { return readBytes == 0 ? - 1 : readBytes ; } readBytes += loaded ; } return readBytes ; }


Input: Orchestrate what definitions should be used for this bundle . [CODESPLIT] private List < BundleContentGeneratorDefinition > getRequestedDefinitions ( List < String > generators ) { Stream < BundleContentGeneratorDefinition > stream = definitions . stream ( ) ; if ( generators == null || generators . isEmpty ( ) ) { // Filter out default generators stream = stream . filter ( BundleContentGeneratorDefinition :: isEnabledByDefault ) ; } else { stream = stream . filter ( def -> generators . contains ( def . getId ( ) ) ) ; } return stream . sorted ( Comparator . comparingInt ( BundleContentGeneratorDefinition :: getOrder ) ) . collect ( Collectors . toList ( ) ) ; }


Input: From https : // stackoverflow . com / a / 31928740 / 33905 [CODESPLIT] @ VisibleForTesting protected static Map < String , String [ ] > getQueryParameters ( HttpServletRequest request ) { Map < String , String [ ] > queryParameters = new HashMap <> ( ) ; String queryString = request . getQueryString ( ) ; if ( StringUtils . isEmpty ( queryString ) ) { return queryParameters ; } String [ ] parameters = queryString . split ( "&" ) ; for ( String parameter : parameters ) { String [ ] keyValuePair = parameter . split ( "=" ) ; String [ ] values = queryParameters . get ( keyValuePair [ 0 ] ) ; values = ArrayUtils . add ( values , keyValuePair . length == 1 ? "" : keyValuePair [ 1 ] ) ; //length is one if no value is available. queryParameters . put ( keyValuePair [ 0 ] , values ) ; } return queryParameters ; }


Input: This is used by destination which performs CRUD operations . Records sent from MySQLBinLog origin have / Data field that contains a map of key - value for INSERT and UPDATE operation but / OldData field for DELETE operation . This method looks into / OldData for DELETE operation so that users don t need a separate field - column mapping for DELETE . [CODESPLIT] @ Override public String getFieldPath ( String fieldPath , int operation ) { if ( operation == OperationType . DELETE_CODE ) { return fieldPath . replace ( DATA_FIELD , OLDDATA_FIELD ) ; } return fieldPath ; }


Input: If there is a RecordEL then an arg could eval to empty string . This method returns args that are not null or empty . [CODESPLIT] private String [ ] getNonEmptyArgs ( List < String > appArgs ) { List < String > nonEmpty = new ArrayList <> ( ) ; appArgs . forEach ( ( String val ) -> { if ( ! StringUtils . isEmpty ( val ) ) { nonEmpty . add ( val ) ; } } ) ; return nonEmpty . toArray ( new String [ nonEmpty . size ( ) ] ) ; }


Input: Return Operation based on the operation code . If the code has a number that Kudu destination doesn t support it throws UnsupportedOperationException . [CODESPLIT] protected Operation getOperation ( KuduTable table , int op ) throws UnsupportedOperationException { Operation operation = null ; switch ( op ) { case OperationType . INSERT_CODE : operation = table . newInsert ( ) ; break ; case OperationType . UPSERT_CODE : operation = table . newUpsert ( ) ; break ; case OperationType . UPDATE_CODE : operation = table . newUpdate ( ) ; break ; case OperationType . DELETE_CODE : operation = table . newDelete ( ) ; break ; default : LOG . error ( "Operation {} not supported" , op ) ; throw new UnsupportedOperationException ( String . format ( "Unsupported Operation: %s" , op ) ) ; } return operation ; }


Input: Convert the bytes to a human readable format upto 2 decimal places The maximum unit is TB so anything exceeding 1024 TB will be shown with TB unit . [CODESPLIT] static String convertBytesToDisplayFormat ( double bytes ) { int unitIdx = 0 ; double unitChangedBytes = bytes ; while ( unitIdx < UNITS . length - 1 && Math . floor ( unitChangedBytes / 1024 ) > 0 ) { unitChangedBytes = unitChangedBytes / 1024 ; unitIdx ++ ; } return df . format ( unitChangedBytes ) + " " + UNITS [ unitIdx ] ; }


Input: Scans the directory of for the next file . [CODESPLIT] public LiveFile scan ( LiveFile current ) throws IOException { try { return scanInternal ( current ) ; } catch ( NoSuchFileException ex ) { // this could happen because there has been a file rotation/deletion after the search/filter/sort and before // the creation of the nen current. Lets sleep for 50ms and try again, if fails again give up. ThreadUtil . sleep ( 50 ) ; return scanInternal ( current ) ; } }


Input: Scans the directory for number of files yet to be processed . [CODESPLIT] public long getPendingFiles ( LiveFile current ) throws IOException { //Current will not be acceptable for roll files (if active file is without a counter/date pattern) //and will be later renamed to a file with counter/date suffix, if that is the case we should //return 0 as number of pending files if ( current == null || rollMode . isCurrentAcceptable ( current . getPath ( ) . getFileName ( ) . toString ( ) ) ) { return findToBeProcessedMatchingFiles ( current != null ? current . refresh ( ) : null ) . size ( ) ; } return 0 ; }


Input: Return true if this dependency and given set of versions is whitelisted . [CODESPLIT] public static boolean isWhitelisted ( String name , Properties specificWhitelist , Map < String , List < Dependency > > dependencies ) { if ( specificWhitelist != null && specificWhitelist . containsKey ( name ) ) { return versionsMatch ( specificWhitelist . getProperty ( name ) , dependencies . keySet ( ) ) ; } // Otherwise try hardcoded rules: WhitelistRule rule = WHITELIST_RULES . get ( name ) ; return rule != null && rule . isWhitelisted ( dependencies ) ; }


Input: Compare expected versions with given versions to see if they are the same or not . [CODESPLIT] private static boolean versionsMatch ( String expectedVersions , Set < String > versions ) { Set < String > expectedSet = Sets . newHashSet ( expectedVersions . split ( "," ) ) ; return Sets . symmetricDifference ( expectedSet , versions ) . isEmpty ( ) ; }


Input: Validates the parameters for this config bean . [CODESPLIT] public void init ( Stage . Context context , String groupName , String prefix , List < Stage . ConfigIssue > issues ) { if ( ! trustStorePath . isEmpty ( ) ) { if ( Files . notExists ( Paths . get ( trustStorePath ) ) ) { issues . add ( context . createConfigIssue ( groupName , prefix + "trustStorePath" , HTTP_04 , trustStorePath ) ) ; } try { if ( trustStorePassword . get ( ) . isEmpty ( ) ) { issues . add ( context . createConfigIssue ( groupName , prefix + "trustStorePassword" , HTTP_05 ) ) ; } } catch ( StageException e ) { issues . add ( context . createConfigIssue ( groupName , prefix + "trustStorePassword" , HTTP_29 , e . toString ( ) ) ) ; } } if ( ! keyStorePath . isEmpty ( ) ) { if ( Files . notExists ( Paths . get ( keyStorePath ) ) ) { issues . add ( context . createConfigIssue ( groupName , prefix + "keyStorePath" , HTTP_04 , keyStorePath ) ) ; } try { if ( keyStorePassword . get ( ) . isEmpty ( ) ) { issues . add ( context . createConfigIssue ( groupName , prefix + "keyStorePassword" , HTTP_05 ) ) ; } } catch ( StageException e ) { issues . add ( context . createConfigIssue ( groupName , prefix + "keyStorePassword" , HTTP_29 , e . toString ( ) ) ) ; } } }


Input: Returns true if given snapshot output is usable - e . g . if it make sense to persist . [CODESPLIT] public static boolean isSnapshotOutputUsable ( List < StageOutput > stagesOutput ) { // In case that the snapshot actually does not exists if ( stagesOutput == null ) { return false ; } // We're looking for at least one output lane that is not empty. In most cases the first stage in the list will // be origin that generated some data and hence the loop will terminate fast. In the worst case scenario we will // iterate over all stages in attempt to find at least one record in the snapshot. for ( StageOutput output : stagesOutput ) { if ( CollectionUtils . isNotEmpty ( output . getErrorRecords ( ) ) || CollectionUtils . isNotEmpty ( output . getEventRecords ( ) ) || CollectionUtils . isNotEmpty ( output . getStageErrors ( ) ) ) { return true ; } for ( Map . Entry < String , List < Record > > entry : output . getOutput ( ) . entrySet ( ) ) { if ( ! entry . getValue ( ) . isEmpty ( ) ) { return true ; } } } return false ; }


Input: Bootstrapping the Driver which starts a Spark job on cluster [CODESPLIT] public static void main ( String [ ] args ) throws Exception { SparkStreamingBinding binding = null ; try { binding = SparkStreamingBindingFactory . build ( BootstrapCluster . getProperties ( ) ) ; binding . init ( ) ; BootstrapCluster . createTransformers ( binding . getStreamingContext ( ) . sparkContext ( ) , binding . getSparkSession ( ) ) ; binding . startContext ( ) ; binding . awaitTermination ( ) ; } catch ( Throwable error ) { String msg = "Error trying to invoke BootstrapClusterStreaming.main: " + error ; System . err . println ( new Date ( ) + ": " + msg ) ; error . printStackTrace ( System . err ) ; // required as in local mode the following seems to be lost LOG . error ( msg , error ) ; throw new IllegalStateException ( msg , error ) ; } finally { try { if ( binding != null ) { binding . close ( ) ; } } catch ( Exception ex ) { LOG . warn ( "Error on binding close: " + ex , ex ) ; } } }


Input: { [CODESPLIT] @ Override protected List < ConfigIssue > init ( ) { // Validate configuration values and open any required resources. List < ConfigIssue > issues = super . init ( ) ; Optional . ofNullable ( conf . init ( getContext ( ) , CONF_PREFIX ) ) . ifPresent ( issues :: addAll ) ; try { ConnectorConfig partnerConfig = ForceUtils . getPartnerConfig ( conf , new WaveSessionRenewer ( ) ) ; connection = Connector . newConnection ( partnerConfig ) ; LOG . info ( "Successfully authenticated as {}" , conf . username ) ; if ( conf . mutualAuth . useMutualAuth ) { ForceUtils . setupMutualAuth ( partnerConfig , conf . mutualAuth ) ; } String soapEndpoint = connection . getConfig ( ) . getServiceEndpoint ( ) ; restEndpoint = soapEndpoint . substring ( 0 , soapEndpoint . indexOf ( "services/Soap/" ) ) ; httpClient = new HttpClient ( ForceUtils . makeSslContextFactory ( conf ) ) ; if ( conf . useProxy ) { ForceUtils . setProxy ( httpClient , conf ) ; } httpClient . start ( ) ; } catch ( Exception e ) { LOG . error ( "Exception during init()" , e ) ; issues . add ( getContext ( ) . createConfigIssue ( Groups . FORCE . name ( ) , ForceConfigBean . CONF_PREFIX + "authEndpoint" , Errors . WAVE_00 , ForceUtils . getExceptionCode ( e ) + ", " + ForceUtils . getExceptionMessage ( e ) ) ) ; } // If issues is not empty, the UI will inform the user of each configuration issue in the list. return issues ; }


Input: { [CODESPLIT] @ Override public void write ( Batch batch ) throws StageException { if ( batch . getRecords ( ) . hasNext ( ) ) { if ( datasetID == null ) { LOG . info ( "Opening dataset" ) ; try { openDataset ( ) ; } catch ( ConnectionException ce ) { throw new StageException ( Errors . WAVE_01 , ce ) ; } } LOG . info ( "Writing batch to dataset" ) ; writeToDataset ( batch ) ; try { commitDataset ( ) ; } catch ( ConnectionException | IOException e ) { throw new StageException ( Errors . WAVE_01 , e ) ; } } }


Input: We have special type of a ConfigDef called RUNTIME . This config is never displayed in UI and instead it s values are supplied at runtime . This method is the runtime method that propagates them . [CODESPLIT] private void propagateRuntimeConfiguration ( ) { // If pipeline wasn't loaded or there if there are no stages, there is nothing to propagate if ( pipelineBean == null || pipelineBean . getPipelineStageBeans ( ) == null ) { return ; } for ( StageBean stageBean : pipelineBean . getPipelineStageBeans ( ) . getStages ( ) ) { for ( ServiceDependencyDefinition serviceDependency : stageBean . getDefinition ( ) . getServices ( ) ) { ServiceBean stageService = stageBean . getService ( serviceDependency . getService ( ) ) ; if ( stageService == null ) { continue ; } ServiceConfiguration serviceConfiguration = stageService . getConf ( ) ; List < Config > configs = serviceConfiguration . getConfiguration ( ) ; // Simply remove all RUNTIME configs configs . removeAll ( serviceDependency . getConfiguration ( ) . keySet ( ) . stream ( ) . map ( serviceConfiguration :: getConfig ) . collect ( Collectors . toList ( ) ) ) ; // And insert them with the stage-instance-constant values serviceDependency . getConfiguration ( ) . forEach ( ( key , value ) -> configs . add ( new Config ( key , value ) ) ) ; // And overwrite the new state serviceConfiguration . setConfig ( configs ) ; } } }


Input: { [CODESPLIT] @ Override protected List < ConfigIssue > init ( ) { // Validate configuration values and open any required resources. List < ConfigIssue > issues = super . init ( ) ; if ( issues . isEmpty ( ) ) { jdbcUtil = UtilsProvider . getJdbcUtil ( ) ; } errorRecordHandler = new DefaultErrorRecordHandler ( getContext ( ) ) ; Processor . Context context = getContext ( ) ; queryEval = getContext ( ) . createELEval ( "query" ) ; issues = hikariConfigBean . validateConfigs ( context , issues ) ; if ( context . getRunnerId ( ) == 0 ) { if ( issues . isEmpty ( ) && null == dataSource ) { try { dataSource = jdbcUtil . createDataSourceForRead ( hikariConfigBean ) ; context . getStageRunnerSharedMap ( ) . put ( "jdbcLookupProcessor.dataSource" , dataSource ) ; } catch ( StageException e ) { issues . add ( context . createConfigIssue ( Groups . JDBC . name ( ) , CONNECTION_STRING , JdbcErrors . JDBC_00 , e . toString ( ) ) ) ; } } } else { dataSource = ( HikariDataSource ) context . getStageRunnerSharedMap ( ) . get ( "jdbcLookupProcessor.dataSource" ) ; } if ( issues . isEmpty ( ) ) { this . defaultValue = calculateDefault ( context , issues ) ; } if ( issues . isEmpty ( ) ) { cache = buildCache ( ) ; cacheCleaner = new CacheCleaner ( cache , "JdbcLookupProcessor" , 10 * 60 * 1000 ) ; if ( cacheConfig . enabled ) { preprocessThreads = Math . min ( hikariConfigBean . minIdle , Runtime . getRuntime ( ) . availableProcessors ( ) - 1 ) ; preprocessThreads = Math . max ( preprocessThreads , 1 ) ; } } if ( context . getRunnerId ( ) == 0 ) { if ( issues . isEmpty ( ) && generationExecutor == null ) { generationExecutor = new SafeScheduledExecutorService ( hikariConfigBean . maximumPoolSize , "JDBC Lookup Cache Warmer" ) ; context . getStageRunnerSharedMap ( ) . put ( "jdbcLookupProcessor.generationExecutor" , generationExecutor ) ; } } else { generationExecutor = ( SafeScheduledExecutorService ) context . getStageRunnerSharedMap ( ) . get ( "jdbcLookupProcessor.generationExecutor" ) ; } // If issues is not empty, the UI will inform the user of each configuration issue in the list. return issues ; }


Input: { [CODESPLIT] @ Override public void destroy ( ) { if ( getContext ( ) . getRunnerId ( ) == 0 ) { if ( generationExecutor != null ) { generationExecutor . shutdown ( ) ; try { if ( ! generationExecutor . awaitTermination ( 5 , TimeUnit . SECONDS ) ) { generationExecutor . shutdownNow ( ) ; } } catch ( InterruptedException ex ) { LOG . error ( "Interrupted while attempting to shutdown Generator Executor: " , ex ) ; Thread . currentThread ( ) . interrupt ( ) ; } } // close dataSource after closing threadpool executor as we could have queries running before closing the executor if ( jdbcUtil != null ) { jdbcUtil . closeQuietly ( dataSource ) ; } } super . destroy ( ) ; }


Input: { [CODESPLIT] @ Override protected void process ( Record record , SingleLaneBatchMaker batchMaker ) throws StageException { try { ELVars elVars = getContext ( ) . createELVars ( ) ; RecordEL . setRecordInContext ( elVars , record ) ; String preparedQuery = queryEval . eval ( elVars , query , String . class ) ; Optional < List < Map < String , Field > > > entry = cache . get ( preparedQuery ) ; if ( ! entry . isPresent ( ) ) { // No results switch ( missingValuesBehavior ) { case SEND_TO_ERROR : LOG . error ( JdbcErrors . JDBC_04 . getMessage ( ) , preparedQuery ) ; errorRecordHandler . onError ( new OnRecordErrorException ( record , JdbcErrors . JDBC_04 , preparedQuery ) ) ; break ; case PASS_RECORD_ON : batchMaker . addRecord ( record ) ; break ; default : throw new IllegalStateException ( "Unknown missing value behavior: " + missingValuesBehavior ) ; } } else { List < Map < String , Field > > values = entry . get ( ) ; switch ( multipleValuesBehavior ) { case FIRST_ONLY : setFieldsInRecord ( record , values . get ( 0 ) ) ; batchMaker . addRecord ( record ) ; break ; case SPLIT_INTO_MULTIPLE_RECORDS : for ( Map < String , Field > lookupItem : values ) { Record newRecord = getContext ( ) . cloneRecord ( record ) ; setFieldsInRecord ( newRecord , lookupItem ) ; batchMaker . addRecord ( newRecord ) ; } break ; case ALL_AS_LIST : Map < String , List < Field > > valuesMap = new HashMap <> ( ) ; for ( Map < String , Field > lookupItem : values ) { lookupItem . forEach ( ( k , v ) -> { if ( valuesMap . get ( k ) == null ) { List < Field > lookupValue = new ArrayList <> ( ) ; valuesMap . put ( k , lookupValue ) ; } valuesMap . get ( k ) . add ( v ) ; } ) ; } Map < String , Field > valueMap = new HashMap <> ( ) ; valuesMap . forEach ( ( k , v ) -> valueMap . put ( k , Field . create ( v ) ) ) ; setFieldsInRecord ( record , valueMap ) ; batchMaker . addRecord ( record ) ; break ; default : throw new IllegalStateException ( "Unknown multiple value behavior: " + multipleValuesBehavior ) ; } } } catch ( ELEvalException e ) { LOG . error ( JdbcErrors . JDBC_01 . getMessage ( ) , query , e ) ; throw new OnRecordErrorException ( record , JdbcErrors . JDBC_01 , query ) ; } catch ( ExecutionException e ) { Throwables . propagateIfPossible ( e . getCause ( ) , StageException . class ) ; throw new IllegalStateException ( e ) ; // The cache loader shouldn't throw anything that isn't a StageException. } catch ( OnRecordErrorException error ) { // NOSONAR errorRecordHandler . onError ( new OnRecordErrorException ( record , error . getErrorCode ( ) , error . getParams ( ) ) ) ; } }


Input: Validate Ominture Report Description . [CODESPLIT] private void validateReportDescription ( List < ConfigIssue > issues ) { if ( ! jsonMapper . isValidJson ( this . reportDescription ) ) { issues . add ( getContext ( ) . createConfigIssue ( Groups . REPORT . name ( ) , "reportDescription" , Errors . OMNITURE_03 ) ) ; } }


Input: / * Escapes special characters with a preceding slash [CODESPLIT] private String escapeValue ( String str , boolean escapeSpace ) { int len = str . length ( ) ; int bufLen = len * 2 ; if ( bufLen < 0 ) { bufLen = Integer . MAX_VALUE ; } StringBuilder outBuffer = new StringBuilder ( bufLen ) ; for ( int x = 0 ; x < len ; x ++ ) { char aChar = str . charAt ( x ) ; // Handle common case first, selecting largest block that // avoids the specials below if ( ( aChar > 61 ) && ( aChar < 127 ) ) { if ( aChar == ' ' ) { outBuffer . append ( ' ' ) ; outBuffer . append ( ' ' ) ; continue ; } outBuffer . append ( aChar ) ; continue ; } switch ( aChar ) { case ' ' : if ( x == 0 || escapeSpace ) outBuffer . append ( ' ' ) ; outBuffer . append ( ' ' ) ; break ; case ' ' : outBuffer . append ( ' ' ) ; outBuffer . append ( ' ' ) ; break ; case ' ' : outBuffer . append ( ' ' ) ; outBuffer . append ( ' ' ) ; break ; case ' ' : outBuffer . append ( ' ' ) ; outBuffer . append ( ' ' ) ; break ; case ' ' : outBuffer . append ( ' ' ) ; outBuffer . append ( ' ' ) ; break ; case ' ' : // Fall through case ' ' : // Fall through case ' ' : // Fall through case ' ' : outBuffer . append ( ' ' ) ; outBuffer . append ( aChar ) ; break ; default : outBuffer . append ( aChar ) ; } } return outBuffer . toString ( ) ; }


Input: Called by JdbcTarget [CODESPLIT] public static JdbcRecordWriter createJdbcRecordWriter ( String connectionString , HikariDataSource dataSource , String schema , String tableName , List < JdbcFieldColumnParamMapping > customMappings , boolean rollbackOnError , boolean useMultiRowOp , int maxPrepStmtParameters , int defaultOpCode , UnsupportedOperationAction unsupportedAction , DuplicateKeyAction duplicateKeyAction , JdbcRecordReader recordReader , boolean caseSensitive , List < String > customDataSqlStateCodes ) throws StageException { if ( defaultOpCode == OperationType . LOAD_CODE ) { return new JdbcLoadRecordWriter ( connectionString , dataSource , schema , tableName , customMappings , duplicateKeyAction , recordReader , caseSensitive , customDataSqlStateCodes ) ; } else { return createJdbcRecordWriter ( connectionString , dataSource , schema , tableName , customMappings , null , rollbackOnError , useMultiRowOp , maxPrepStmtParameters , defaultOpCode , unsupportedAction , recordReader , caseSensitive , customDataSqlStateCodes ) ; } }


Input: Called by JdbcTeeProcessor [CODESPLIT] public static JdbcRecordWriter createJdbcRecordWriter ( String connectionString , HikariDataSource dataSource , String schema , String tableName , List < JdbcFieldColumnParamMapping > customMappings , List < JdbcFieldColumnMapping > generatedColumnMappings , boolean rollbackOnError , boolean useMultiRowOp , int maxPrepStmtParameters , int defaultOpCode , UnsupportedOperationAction unsupportedAction , JdbcRecordReader recordReader , boolean caseSensitive , List < String > customDataSqlStateCodes ) throws StageException { JdbcRecordWriter recordWriter ; if ( useMultiRowOp ) { recordWriter = new JdbcMultiRowRecordWriter ( connectionString , dataSource , schema , tableName , rollbackOnError , customMappings , maxPrepStmtParameters , defaultOpCode , unsupportedAction , generatedColumnMappings , recordReader , caseSensitive , customDataSqlStateCodes ) ; } else { recordWriter = new JdbcGenericRecordWriter ( connectionString , dataSource , schema , tableName , rollbackOnError , customMappings , defaultOpCode , unsupportedAction , generatedColumnMappings , recordReader , caseSensitive , customDataSqlStateCodes ) ; } return recordWriter ; }


Input: Remove implicit field mapping [CODESPLIT] private void upgradeV1toV2 ( List < Config > configs ) { configs . removeIf ( config -> ( config . getName ( ) . equals ( IMPLICIT_FIELD_MAPPING_CONFIG ) || config . getName ( ) . equals ( BIG_QUERY_IMPLICIT_FIELD_MAPPING_CONFIG ) ) ) ; configs . add ( new Config ( MAX_CACHE_SIZE , - 1 ) ) ; }


Input: Extracts named groups from the raw data [CODESPLIT] public Map < String , String > extractNamedGroups ( final CharSequence rawData ) { Matcher matcher = compiledPattern . matcher ( rawData ) ; if ( matcher . find ( ) ) { MatchResult r = matcher . toMatchResult ( ) ; if ( r != null && r . namedGroups ( ) != null ) { return r . namedGroups ( ) ; } } return null ; }


Input: If a null value is passed to this method it s replaced with a dummy due to the fact the payload for each message is wrapped in an Optional . [CODESPLIT] public void consumerCommit ( String offset ) { Object offsetValue = offset ; if ( offsetValue == null ) { offsetValue = new NullOffset ( ) ; } LOG . trace ( "Commit Offset: '{}'" , offsetValue ) ; try { producerQueue . put ( new Message ( MessageType . CONSUMER_COMMIT , offsetValue ) ) ; } catch ( InterruptedException e ) { LOG . info ( "Interrupted while queuing '{}'" , MessageType . CONSUMER_COMMIT . name ( ) , offsetValue ) ; Thread . currentThread ( ) . interrupt ( ) ; } }


Input: Creates an instance of SDC and adds to pool [CODESPLIT] @ SuppressWarnings ( "unchecked" ) protected EmbeddedSDC create ( ) throws Exception { Utils . checkState ( open , "Not open" ) ; final EmbeddedSDC embeddedSDC = new EmbeddedSDC ( ) ; Object source ; // post-batch runnable Object pipelineStartResult = BootstrapCluster . startPipeline ( ( ) -> LOG . debug ( "Batch completed" ) ) ; source = pipelineStartResult . getClass ( ) . getDeclaredField ( "source" ) . get ( pipelineStartResult ) ; if ( source instanceof DSource ) { long startTime = System . currentTimeMillis ( ) ; long endTime = startTime ; long diff = 0 ; Source actualSource = ( ( DSource ) source ) . getSource ( ) ; while ( actualSource == null && diff < 60000 ) { Thread . sleep ( 100 ) ; actualSource = ( ( DSource ) source ) . getSource ( ) ; endTime = System . currentTimeMillis ( ) ; diff = endTime - startTime ; } if ( actualSource == null ) { throw new IllegalStateException ( "Actual source is null, pipeline may not have been initialized" ) ; } source = actualSource ; } if ( ! ( source instanceof ClusterSource ) ) { throw new IllegalArgumentException ( "Source is not of type ClusterSource: " + source . getClass ( ) . getName ( ) ) ; } embeddedSDC . setSource ( ( ClusterSource ) source ) ; embeddedSDC . setSparkProcessors ( ( List < Object > ) pipelineStartResult . getClass ( ) . getDeclaredField ( "sparkProcessors" ) . get ( pipelineStartResult ) ) ; return embeddedSDC ; }


Input: What is this?! Spark could end up scheduling transformers to run on machines that may actually not have received the data due to : - Non - local reads : Spark Locality is time - bound so if local executor where the data is located is not available Spark will tell another one to pick up the task . - Shuffle in the transformer : After a shuffle it is possible the next task passing the data to the next batch could end up on a different executor . [CODESPLIT] private EmbeddedSDC fastForward ( int id ) throws Exception { if ( id == sparkProcessorCount ) { return null ; } LOG . info ( "No SDC was found at ID: " + id + ". Fast-forwarding.." ) ; EmbeddedSDC sdc ; // If there are not SDCs that are just idling, create a new one, else return one from the not started pool. if ( notStarted . isEmpty ( ) ) { sdc = create ( ) ; } else { sdc = getNotStartedSDC ( ) ; } Class < ? > clusterFunctionClass = Class . forName ( "com.streamsets.pipeline.cluster.ClusterFunctionImpl" ) ; Method getBatch = clusterFunctionClass . getMethod ( "getNextBatch" , int . class , EmbeddedSDC . class ) ; Method forward = clusterFunctionClass . getMethod ( "doForward" , Iterator . class , int . class , EmbeddedSDC . class ) ; sdc . getSource ( ) . put ( Collections . emptyList ( ) ) ; getBatch . invoke ( null , 0 , sdc ) ; for ( int i = 0 ; i <= id - 1 ; i ++ ) { forward . invoke ( null , Collections . emptyIterator ( ) , i , sdc ) ; } return sdc ; }


Input: Set the lookup reuslt in the result field [CODESPLIT] private void setFieldsInRecord ( Record record , Map < String , Field > fields ) { record . set ( configBean . resultField , Field . createListMap ( new LinkedHashMap <> ( fields ) ) ) ; }