Input: Serializes a { @link BidResponse } to JSON streamed to a { @link OutputStream } . [CODESPLIT] public void writeBidResponse ( BidResponse resp , OutputStream os ) throws IOException { JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeBidResponse ( resp , gen ) ; }


Input: Serializes a { @link BidResponse } to JSON streamed to a { @link Writer } . [CODESPLIT] public void writeBidResponse ( BidResponse resp , Writer writer ) throws IOException { JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeBidResponse ( resp , gen ) ; }


Input: Serializes a { [CODESPLIT] public final void writeBidResponse ( BidResponse resp , JsonGenerator gen ) throws IOException { gen . writeStartObject ( ) ; writeBidResponseFields ( resp , gen ) ; writeExtensions ( resp , gen ) ; gen . writeEndObject ( ) ; gen . flush ( ) ; }


Input: @return The OpenRTB SeatBid with the specified ID ; will be created if not existent . The ID should be present in the request s wseat . [CODESPLIT] public static SeatBid . Builder seatBid ( BidResponse . Builder response , String seat ) { checkNotNull ( seat ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( seatbid . hasSeat ( ) && seat . equals ( seatbid . getSeat ( ) ) ) { return seatbid ; } } return response . addSeatbidBuilder ( ) . setSeat ( seat ) ; }


Input: Iterates all bids . [CODESPLIT] public static Iterable < Bid . Builder > bids ( BidResponse . Builder response ) { return new ResponseBidsIterator ( response , SEAT_ANY , null ) ; }


Input: Iterates all bids from a specific seat . [CODESPLIT] public static List < Bid . Builder > bids ( BidResponse . Builder response , @ Nullable String seatFilter ) { for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( filterSeat ( seatbid , seatFilter ) ) { return seatbid . getBidBuilderList ( ) ; } } return ImmutableList . of ( ) ; }


Input: Finds a bid by ID . [CODESPLIT] @ Nullable public static Bid . Builder bidWithId ( BidResponse . Builder response , String id ) { checkNotNull ( id ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { for ( Bid . Builder bid : seatbid . getBidBuilderList ( ) ) { if ( id . equals ( bid . getId ( ) ) ) { return bid ; } } } return null ; }


Input: Finds bids by a custom criteria . [CODESPLIT] public static Stream < Bid . Builder > bidStreamWith ( BidResponse . Builder response , @ Nullable String seatFilter , @ Nullable Predicate < Bid . Builder > bidFilter ) { return StreamSupport . stream ( new ResponseBidsIterator ( response , seatFilter , bidFilter ) . spliterator ( ) , false ) ; }


Input: Finds bids by a custom criteria . [CODESPLIT] public static Iterable < Bid . Builder > bidsWith ( BidResponse . Builder response , @ Nullable String seatFilter , @ Nullable Predicate < Bid . Builder > bidFilter ) { return new ResponseBidsIterator ( response , seatFilter , bidFilter ) ; }


Input: Updates bids from all seats . [CODESPLIT] public static boolean updateBids ( BidResponse . Builder response , Function < Bid . Builder , Boolean > updater ) { checkNotNull ( updater ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { updated |= ProtoUtils . update ( seatbid . getBidBuilderList ( ) , updater ) ; } return updated ; }


Input: Updates bids from a given seat . [CODESPLIT] public static boolean updateBids ( BidResponse . Builder response , @ Nullable String seatFilter , Function < Bid . Builder , Boolean > updater ) { checkNotNull ( updater ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( filterSeat ( seatbid , seatFilter ) ) { return ProtoUtils . update ( seatbid . getBidBuilderList ( ) , updater ) ; } } return false ; }


Input: Remove bids by bid . [CODESPLIT] public static boolean removeBids ( BidResponse . Builder response , Predicate < Bid . Builder > filter ) { checkNotNull ( filter ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { updated |= removeBids ( seatbid , filter ) ; } return updated ; }


Input: Remove bids by seat and bid . [CODESPLIT] public static boolean removeBids ( BidResponse . Builder response , @ Nullable String seatFilter , Predicate < Bid . Builder > bidFilter ) { checkNotNull ( bidFilter ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { if ( filterSeat ( seatbid , seatFilter ) ) { updated |= removeBids ( seatbid , bidFilter ) ; } } return updated ; }


Input: Finds an { @link Imp } by ID . [CODESPLIT] @ Nullable public static Imp impWithId ( BidRequest request , String id ) { checkNotNull ( id ) ; for ( Imp imp : request . getImpList ( ) ) { if ( imp . getId ( ) . equals ( id ) ) { return imp ; } } return null ; }


Input: Find an { @link Imp } by its ID and its { @link Banner } s ID . [CODESPLIT] @ Nullable public static Imp bannerImpWithId ( BidRequest request , @ Nullable String impId , String bannerId ) { checkNotNull ( bannerId ) ; for ( Imp imp : request . getImpList ( ) ) { if ( ( impId == null || imp . getId ( ) . equals ( impId ) ) && imp . hasBanner ( ) && imp . getBanner ( ) . getId ( ) . equals ( bannerId ) ) { return imp ; } } return null ; }


Input: Optimized code for most filtered lookups . This is worth the effort because bidder code may invoke these lookup methods intensely ; common cases like everything - filtered or nothing - filtered are very dominant ; and simpler code previously used needed lots of temporary collections . [CODESPLIT] public static Iterable < Imp > impsWith ( BidRequest request , Predicate < Imp > impFilter ) { checkNotNull ( impFilter ) ; List < Imp > imps = request . getImpList ( ) ; if ( imps . isEmpty ( ) || impFilter == IMP_ALL ) { return imps ; } else if ( impFilter == IMP_NONE ) { return ImmutableList . of ( ) ; } boolean included = impFilter . test ( imps . get ( 0 ) ) ; int size = imps . size ( ) , i ; for ( i = 1 ; i < size ; ++ i ) { if ( impFilter . test ( imps . get ( i ) ) != included ) { break ; } } if ( i == size ) { return included ? imps // Unmodifiable, comes from protobuf : ImmutableList . < Imp > of ( ) ; } int headingSize = i ; return new FluentIterable < Imp > ( ) { @ Override public Iterator < Imp > iterator ( ) { Iterator < Imp > unfiltered = imps . iterator ( ) ; return new AbstractIterator < Imp > ( ) { private int heading = 0 ; @ Override protected Imp computeNext ( ) { while ( unfiltered . hasNext ( ) ) { Imp imp = unfiltered . next ( ) ; if ( ( heading ++ < headingSize ) ? included : impFilter . test ( imp ) ) { return imp ; } } return endOfData ( ) ; } } ; } } ; }


Input: Adds impression type subfilters to a base filter to further restricts impressions that contain a banner video and / or native object . [CODESPLIT] public static Predicate < Imp > addFilters ( Predicate < Imp > baseFilter , boolean banner , boolean video , boolean nativ ) { int orCount = ( banner ? 1 : 0 ) + ( video ? 1 : 0 ) + ( nativ ? 1 : 0 ) ; if ( baseFilter == IMP_NONE || orCount == 0 ) { return baseFilter ; } Predicate < Imp > typeFilter = null ; if ( banner ) { typeFilter = Imp :: hasBanner ; } if ( video ) { typeFilter = typeFilter == null ? Imp :: hasVideo : typeFilter . or ( Imp :: hasVideo ) ; } if ( nativ ) { typeFilter = typeFilter == null ? Imp :: hasNative : typeFilter . or ( Imp :: hasNative ) ; } return baseFilter == IMP_ALL ? typeFilter : baseFilter . and ( typeFilter ) ; }


Input: Performs a filter by seat . [CODESPLIT] public static boolean filterSeat ( SeatBidOrBuilder seatbid , @ Nullable String seatFilter ) { return seatFilter == null ? ! seatbid . hasSeat ( ) : seatFilter == SEAT_ANY || seatFilter . equals ( seatbid . getSeat ( ) ) ; }


Input: Serializes a { [CODESPLIT] public String writeNativeRequest ( NativeRequest req ) throws IOException { try ( StringWriter writer = new StringWriter ( ) ) { writeNativeRequest ( req , writer ) ; return writer . toString ( ) ; } }


Input: Serializes a { @link NativeRequest } to JSON streamed into an { @link Writer } . [CODESPLIT] public void writeNativeRequest ( NativeRequest req , Writer writer ) throws IOException { JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeNativeRequest ( req , gen ) ; }


Input: Serializes a { @link NativeRequest } to JSON streamed into an { @link OutputStream } . [CODESPLIT] public void writeNativeRequest ( NativeRequest req , OutputStream os ) throws IOException { JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeNativeRequest ( req , gen ) ; }


Input: Serializes a { [CODESPLIT] public final void writeNativeRequest ( NativeRequest req , JsonGenerator gen ) throws IOException { gen . writeStartObject ( ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeObjectFieldStart ( "native" ) ; } writeNativeRequestFields ( req , gen ) ; writeExtensions ( req , gen ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeEndObject ( ) ; } gen . writeEndObject ( ) ; gen . flush ( ) ; }


Input: Serializes a { [CODESPLIT] public String writeNativeResponse ( NativeResponse resp ) throws IOException { try ( StringWriter writer = new StringWriter ( ) ) { writeNativeResponse ( resp , writer ) ; return writer . toString ( ) ; } }


Input: Serializes a { @link NativeResponse } to JSON streamed to a { @link OutputStream } . [CODESPLIT] public void writeNativeResponse ( NativeResponse resp , OutputStream os ) throws IOException { JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( os ) ; writeNativeResponse ( resp , gen ) ; }


Input: Serializes a { @link NativeResponse } to JSON streamed to a { @link Writer } . [CODESPLIT] public void writeNativeResponse ( NativeResponse resp , Writer writer ) throws IOException { JsonGenerator gen = factory ( ) . getJsonFactory ( ) . createGenerator ( writer ) ; writeNativeResponse ( resp , gen ) ; }


Input: Serializes a { [CODESPLIT] public final void writeNativeResponse ( NativeResponse resp , JsonGenerator gen ) throws IOException { gen . writeStartObject ( ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeObjectFieldStart ( "native" ) ; } writeNativeResponseFields ( resp , gen ) ; writeExtensions ( resp , gen ) ; if ( factory ( ) . isRootNativeField ( ) ) { gen . writeEndObject ( ) ; } gen . writeEndObject ( ) ; gen . flush ( ) ; }


Input: Read any extensions that may exist in a message . [CODESPLIT] protected final < EB extends ExtendableBuilder < ? , EB > > void readExtensions ( EB msg , JsonParser par ) throws IOException { @ SuppressWarnings ( "unchecked" ) Set < OpenRtbJsonExtReader < EB > > extReaders = factory . getReaders ( ( Class < EB > ) msg . getClass ( ) ) ; if ( extReaders . isEmpty ( ) ) { par . skipChildren ( ) ; return ; } startObject ( par ) ; JsonToken tokLast = par . getCurrentToken ( ) ; JsonLocation locLast = par . getCurrentLocation ( ) ; while ( true ) { boolean extRead = false ; for ( OpenRtbJsonExtReader < EB > extReader : extReaders ) { if ( extReader . filter ( par ) ) { extReader . read ( msg , par ) ; JsonToken tokNew = par . getCurrentToken ( ) ; JsonLocation locNew = par . getCurrentLocation ( ) ; boolean advanced = tokNew != tokLast || ! locNew . equals ( locLast ) ; extRead |= advanced ; if ( ! endObject ( par ) ) { return ; } else if ( advanced && par . getCurrentToken ( ) != JsonToken . FIELD_NAME ) { tokLast = par . nextToken ( ) ; locLast = par . getCurrentLocation ( ) ; } else { tokLast = tokNew ; locLast = locNew ; } } } if ( ! endObject ( par ) ) { // Can't rely on this exit condition inside the for loop because no readers may filter. return ; } if ( ! extRead ) { // No field was consumed by any reader, so we need to skip the field to make progress. if ( logger . isDebugEnabled ( ) ) { logger . debug ( "Extension field not consumed by any reader, skipping: {} @{}:{}" , par . getCurrentName ( ) , locLast . getLineNr ( ) , locLast . getCharOffset ( ) ) ; } par . nextToken ( ) ; par . skipChildren ( ) ; tokLast = par . nextToken ( ) ; locLast = par . getCurrentLocation ( ) ; } // Else loop, try all readers again } }


Input: Special case for empty - string input . Returning null in non - [CODESPLIT] protected final boolean emptyToNull ( JsonParser par ) throws IOException { JsonToken token = par . getCurrentToken ( ) ; if ( token == null ) { token = par . nextToken ( ) ; } return ! factory ( ) . isStrict ( ) && token == null ; }


Input: Register an extension reader . [CODESPLIT] public final < EB extends ExtendableBuilder < ? , EB > > OpenRtbJsonFactory register ( OpenRtbJsonExtReader < EB > extReader , Class < EB > msgKlass ) { extReaders . put ( msgKlass . getName ( ) , extReader ) ; return this ; }


Input: Register an extension writer bound to a specific field name . This writer will be used in preference to a non - field - specific writer that may exist for the same class . [CODESPLIT] public final < T > OpenRtbJsonFactory register ( OpenRtbJsonExtWriter < T > extWriter , Class < T > extKlass , Class < ? extends Message > msgKlass , String fieldName ) { Map < String , Map < String , OpenRtbJsonExtWriter < ? > > > mapMsg = extWriters . get ( msgKlass . getName ( ) ) ; if ( mapMsg == null ) { extWriters . put ( msgKlass . getName ( ) , mapMsg = new LinkedHashMap <> ( ) ) ; } Map < String , OpenRtbJsonExtWriter < ? > > mapKlass = mapMsg . get ( extKlass . getName ( ) ) ; if ( mapKlass == null ) { mapMsg . put ( extKlass . getName ( ) , mapKlass = new LinkedHashMap <> ( ) ) ; } mapKlass . put ( fieldName == null ? FIELDNAME_ALL : fieldName , extWriter ) ; return this ; }


Input: Register an extension writer not bound to any a field name ( so this serializer can be used for any extension of the provided class ) . [CODESPLIT] public final < T > OpenRtbJsonFactory register ( OpenRtbJsonExtWriter < T > extWriter , Class < T > extKlass , Class < ? extends Message > msgKlass ) { return register ( extWriter , extKlass , msgKlass , FIELDNAME_ALL ) ; }


Input: Desserializes a { [CODESPLIT] public NativeRequest readNativeRequest ( CharSequence chars ) throws IOException { return readNativeRequest ( CharSource . wrap ( chars ) . openStream ( ) ) ; }


Input: Desserializes a { [CODESPLIT] public NativeRequest readNativeRequest ( Reader reader ) throws IOException { return ProtoUtils . built ( readNativeRequest ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ; }


Input: Desserializes a { [CODESPLIT] public NativeRequest readNativeRequest ( InputStream is ) throws IOException { try { return ProtoUtils . built ( readNativeRequest ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; } }


Input: Desserializes a { [CODESPLIT] public final NativeRequest . Builder readNativeRequest ( JsonParser par ) throws IOException { if ( emptyToNull ( par ) ) { return null ; } NativeRequest . Builder req = NativeRequest . newBuilder ( ) ; boolean rootNativeField = false ; boolean firstField = true ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { if ( firstField ) { firstField = false ; if ( ( rootNativeField = "native" . equals ( fieldName ) ) == true ) { startObject ( par ) ; fieldName = getCurrentName ( par ) ; par . nextToken ( ) ; } } if ( par . getCurrentToken ( ) != JsonToken . VALUE_NULL ) { readNativeRequestField ( par , req , fieldName ) ; } } } if ( rootNativeField && ! endObject ( par ) ) { par . nextToken ( ) ; } return req ; }


Input: Desserializes a { [CODESPLIT] public NativeResponse readNativeResponse ( CharSequence chars ) throws IOException { return readNativeResponse ( CharSource . wrap ( chars ) . openStream ( ) ) ; }


Input: Desserializes a { [CODESPLIT] public NativeResponse readNativeResponse ( Reader reader ) throws IOException { return ProtoUtils . built ( readNativeResponse ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ; }


Input: Desserializes a { [CODESPLIT] public NativeResponse readNativeResponse ( InputStream is ) throws IOException { try { return ProtoUtils . built ( readNativeResponse ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; } }


Input: Desserializes a { [CODESPLIT] public final NativeResponse . Builder readNativeResponse ( JsonParser par ) throws IOException { if ( emptyToNull ( par ) ) { return null ; } NativeResponse . Builder resp = NativeResponse . newBuilder ( ) ; boolean rootNativeField = false ; boolean firstField = true ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { if ( firstField ) { firstField = false ; if ( ( rootNativeField = "native" . equals ( fieldName ) ) == true ) { startObject ( par ) ; fieldName = getCurrentName ( par ) ; par . nextToken ( ) ; } } if ( par . getCurrentToken ( ) != JsonToken . VALUE_NULL ) { readNativeResponseField ( par , resp , fieldName ) ; } } } if ( rootNativeField && ! endObject ( par ) ) { par . nextToken ( ) ; } return resp ; }


Input: Processes the context s response in - place modifying properties that may contain macros . [CODESPLIT] public void process ( SnippetProcessorContext bidCtx ) { for ( SeatBid . Builder seat : bidCtx . response ( ) . getSeatbidBuilderList ( ) ) { for ( Bid . Builder bid : seat . getBidBuilderList ( ) ) { bidCtx . setBid ( bid ) ; processFields ( bidCtx ) ; } } }


Input: Processes all fields of a bid that should support macro expansion . [CODESPLIT] protected void processFields ( SnippetProcessorContext bidCtx ) { Bid . Builder bid = bidCtx . getBid ( ) ; // Properties that can also be in the RHS of macros used by other properties. if ( extendedFields ) { if ( bid . hasAdid ( ) ) { bid . setAdid ( process ( bidCtx , bid . getAdid ( ) ) ) ; } bid . setId ( process ( bidCtx , bid . getId ( ) ) ) ; } // Properties that are NOT the RHS of any macro. if ( bid . hasAdm ( ) ) { bid . setAdm ( process ( bidCtx , bid . getAdm ( ) ) ) ; } if ( extendedFields ) { if ( bid . hasBurl ( ) ) { bid . setBurl ( process ( bidCtx , bid . getBurl ( ) ) ) ; } if ( bid . hasCid ( ) ) { bid . setCid ( process ( bidCtx , bid . getCid ( ) ) ) ; } if ( bid . hasCrid ( ) ) { bid . setCrid ( process ( bidCtx , bid . getCrid ( ) ) ) ; } if ( bid . hasDealid ( ) ) { bid . setDealid ( process ( bidCtx , bid . getDealid ( ) ) ) ; } bid . setImpid ( process ( bidCtx , bid . getImpid ( ) ) ) ; if ( bid . hasIurl ( ) ) { bid . setIurl ( process ( bidCtx , bid . getIurl ( ) ) ) ; } if ( bid . hasLurl ( ) ) { bid . setIurl ( process ( bidCtx , bid . getLurl ( ) ) ) ; } if ( bid . hasNurl ( ) ) { bid . setNurl ( process ( bidCtx , bid . getNurl ( ) ) ) ; } } }


Input: Desserializes a { [CODESPLIT] public BidRequest readBidRequest ( CharSequence chars ) throws IOException { return readBidRequest ( CharSource . wrap ( chars ) . openStream ( ) ) ; }


Input: Desserializes a { [CODESPLIT] public BidRequest readBidRequest ( Reader reader ) throws IOException { return ProtoUtils . built ( readBidRequest ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ; }


Input: Desserializes a { [CODESPLIT] public BidRequest readBidRequest ( InputStream is ) throws IOException { try { return ProtoUtils . built ( readBidRequest ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; } }


Input: Desserializes a { [CODESPLIT] public final BidRequest . Builder readBidRequest ( JsonParser par ) throws IOException { if ( emptyToNull ( par ) ) { return null ; } BidRequest . Builder req = BidRequest . newBuilder ( ) ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { readBidRequestField ( par , req , fieldName ) ; } } return req ; }


Input: Desserializes a { [CODESPLIT] public BidResponse readBidResponse ( CharSequence chars ) throws IOException { return readBidResponse ( CharSource . wrap ( chars ) . openStream ( ) ) ; }


Input: Desserializes a { [CODESPLIT] public BidResponse readBidResponse ( Reader reader ) throws IOException { return ProtoUtils . built ( readBidResponse ( factory ( ) . getJsonFactory ( ) . createParser ( reader ) ) ) ; }


Input: Desserializes a { [CODESPLIT] public BidResponse readBidResponse ( InputStream is ) throws IOException { try { return ProtoUtils . built ( readBidResponse ( factory ( ) . getJsonFactory ( ) . createParser ( is ) ) ) ; } finally { Closeables . closeQuietly ( is ) ; } }


Input: Desserializes a { [CODESPLIT] public final BidResponse . Builder readBidResponse ( JsonParser par ) throws IOException { if ( emptyToNull ( par ) ) { return null ; } BidResponse . Builder resp = BidResponse . newBuilder ( ) ; for ( startObject ( par ) ; endObject ( par ) ; par . nextToken ( ) ) { String fieldName = getCurrentName ( par ) ; if ( par . nextToken ( ) != JsonToken . VALUE_NULL ) { readBidResponseField ( par , resp , fieldName ) ; } } return resp ; }


Input: Given a message - or - builder returns a message invoking the builder if necessary . [CODESPLIT] @ SuppressWarnings ( "unchecked" ) public static < I extends MessageLiteOrBuilder , O extends MessageLite > O built ( @ Nullable I msg ) { return msg instanceof MessageLite . Builder ? ( O ) ( ( MessageLite . Builder ) msg ) . build ( ) : ( O ) msg ; }


Input: Given a message - or - builder return a builder invoking toBuilder () if necessary . [CODESPLIT] @ SuppressWarnings ( "unchecked" ) public static < I extends MessageLiteOrBuilder , O extends MessageLite . Builder > O builder ( @ Nullable I msg ) { return msg instanceof MessageLite ? ( O ) ( ( MessageLite ) msg ) . toBuilder ( ) : ( O ) msg ; }


Input: Updates every builder from a sequence . [CODESPLIT] public static < B extends MessageLite . Builder > boolean update ( Iterable < B > objs , Function < B , Boolean > updater ) { checkNotNull ( updater ) ; boolean updated = false ; for ( B obj : objs ) { updated |= updater . apply ( obj ) ; } return updated ; }


Input: Runs a filter through a sequence of objects . [CODESPLIT] public static < M extends MessageLiteOrBuilder > List < M > filter ( List < M > objs , Predicate < M > filter ) { checkNotNull ( filter ) ; for ( int i = 0 ; i < objs . size ( ) ; ++ i ) { if ( ! filter . test ( objs . get ( i ) ) ) { // At least one discarded object, go to slow-path. return filterFrom ( objs , filter , i ) ; } } // Optimized common case: all items filtered, return the input sequence. return objs ; }


Input: Returns a copy of a { @link Message } that contains only fields that pass a filter . This will be executed recursively for fields which are child messages . [CODESPLIT] @ Nullable public static < M extends Message > M filter ( M msg , boolean clearEmpty , Predicate < FieldDescriptor > filter ) { checkNotNull ( filter ) ; int i = 0 ; for ( Map . Entry < FieldDescriptor , Object > entry : msg . getAllFields ( ) . entrySet ( ) ) { FieldDescriptor fd = entry . getKey ( ) ; if ( ! filter . test ( fd ) ) { // At least one field discarded, go to slow-path. return filterFrom ( msg , clearEmpty , filter , i , true ) ; } else if ( fd . getType ( ) == FieldDescriptor . Type . MESSAGE ) { // At least one field may have children, go to slow-path. return filterFrom ( msg , clearEmpty , filter , i , false ) ; } ++ i ; } // Optimized common case: all items filtered, return the input sequence. return msg ; }


Input: Returns the current field name or empty string if none . [CODESPLIT] public static String getCurrentName ( JsonParser par ) throws IOException { String name = par . getCurrentName ( ) ; return name == null ? "" : name ; }


Input: Starts an Object skipping the { token and if necessary a field name before it . [CODESPLIT] public static void startObject ( JsonParser par ) throws IOException { JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } if ( token == JsonToken . START_OBJECT ) { par . nextToken ( ) ; } else { throw new JsonParseException ( par , "Expected start of object" ) ; } }


Input: Returns { [CODESPLIT] public static boolean endObject ( JsonParser par ) { JsonToken token = par . getCurrentToken ( ) ; return token != null && token != JsonToken . END_OBJECT ; }


Input: Starts an Array skipping the [ token and if necessary a field name before it . [CODESPLIT] public static void startArray ( JsonParser par ) throws IOException { JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } if ( token == JsonToken . START_ARRAY ) { par . nextToken ( ) ; } else { throw new JsonParseException ( par , "Expected start of array" ) ; } }


Input: Returns { [CODESPLIT] public static boolean endArray ( JsonParser par ) { JsonToken token = par . getCurrentToken ( ) ; return token != null && token != JsonToken . END_ARRAY ; }


Input: Skips a field name if necessary returning the current token then . [CODESPLIT] public static JsonToken peekToken ( JsonParser par ) throws IOException { JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } return token ; }


Input: Skips a field name if necessary returning the current token then which must be the start of an Array or Object : { or [ . [CODESPLIT] public static JsonToken peekStructStart ( JsonParser par ) throws IOException { JsonToken token = peekToken ( par ) ; if ( token . isStructStart ( ) ) { return token ; } else { throw new JsonParseException ( par , "Expected start of array or object" ) ; } }


Input: Writes a boolean as int where false = 0 and true = 1 . [CODESPLIT] public static void writeIntBoolField ( String fieldName , boolean data , JsonGenerator gen ) throws IOException { gen . writeNumberField ( fieldName , data ? 1 : 0 ) ; }


Input: Writes a string array if not empty . [CODESPLIT] public static void writeStrings ( String fieldName , List < String > data , JsonGenerator gen ) throws IOException { if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( String d : data ) { gen . writeString ( d ) ; } gen . writeEndArray ( ) ; } }


Input: Writes an int array if not empty . [CODESPLIT] public static void writeInts ( String fieldName , List < Integer > data , JsonGenerator gen ) throws IOException { if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( Integer d : data ) { gen . writeNumber ( d ) ; } gen . writeEndArray ( ) ; } }


Input: Writes a long using quotes only if it s too big ( over 53 - bit mantissa ) . [CODESPLIT] public static void writeLong ( long data , JsonGenerator gen ) throws IOException { if ( data > MAX_JSON_INT || data < - MAX_JSON_INT ) { gen . writeString ( Long . toString ( data ) ) ; } else { gen . writeNumber ( data ) ; } }


Input: Writes a long using quotes only if it s too big ( over 53 - bit mantissa ) . [CODESPLIT] public static void writeLongField ( String fieldName , long data , JsonGenerator gen ) throws IOException { gen . writeFieldName ( fieldName ) ; writeLong ( data , gen ) ; }


Input: Writes a long array if not empty using quotes for values that are too big . [CODESPLIT] public static void writeLongs ( String fieldName , List < Long > data , JsonGenerator gen ) throws IOException { if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( long d : data ) { writeLong ( d , gen ) ; } gen . writeEndArray ( ) ; } }


Input: Writes a enum value as an int using its Protobuf number . [CODESPLIT] public static void writeEnum ( ProtocolMessageEnum e , JsonGenerator gen ) throws IOException { gen . writeNumber ( e . getNumber ( ) ) ; }


Input: Writes a enum value as an int using its Protobuf number . [CODESPLIT] public static void writeEnumField ( String fieldName , ProtocolMessageEnum e , JsonGenerator gen ) throws IOException { gen . writeNumberField ( fieldName , e . getNumber ( ) ) ; }


Input: Writes a enum array if not empty . [CODESPLIT] public static void writeEnums ( String fieldName , List < ? extends ProtocolMessageEnum > enums , JsonGenerator gen ) throws IOException { if ( ! enums . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( ProtocolMessageEnum e : enums ) { writeEnum ( e , gen ) ; } gen . writeEndArray ( ) ; } }


Input: Reads from either a JSON Value String ( containing CSV ) or a JSON Array . The dual input format is needed because some fields ( e . g . keywords ) were allowed to be of either type in OpenRTB 2 . 2 ; now in 2 . 3 they are all CSV strings only . TODO : Simplify this to only accept CSV strings after 2 . 2 compatibility is dropped . [CODESPLIT] public static String readCsvString ( JsonParser par ) throws IOException { JsonToken currentToken = par . getCurrentToken ( ) ; if ( currentToken == JsonToken . START_ARRAY ) { StringBuilder keywords = new StringBuilder ( ) ; for ( startArray ( par ) ; endArray ( par ) ; par . nextToken ( ) ) { if ( keywords . length ( ) != 0 ) { keywords . append ( ' ' ) ; } keywords . append ( par . getText ( ) ) ; } return keywords . toString ( ) ; } else if ( currentToken == JsonToken . VALUE_STRING ) { return par . getText ( ) ; } else { throw new JsonParseException ( par , "Expected string or array" ) ; } }


Input: Resolve relative URL - s and fix a few java . net . URL errors in handling of URLs with embedded params and pure query targets . [CODESPLIT] public static URL resolveURL ( URL base , String target ) throws MalformedURLException { target = target . trim ( ) ; if ( target . startsWith ( "?" ) ) { return fixPureQueryTargets ( base , target ) ; } return new URL ( base , target ) ; }


Input: Handle the case in RFC3986 section 5 . 4 . 1 example 7 and similar . [CODESPLIT] static URL fixPureQueryTargets ( URL base , String target ) throws MalformedURLException { if ( ! target . startsWith ( "?" ) ) return new URL ( base , target ) ; String basePath = base . getPath ( ) ; String baseRightMost = "" ; int baseRightMostIdx = basePath . lastIndexOf ( "/" ) ; if ( baseRightMostIdx != - 1 ) { baseRightMost = basePath . substring ( baseRightMostIdx + 1 ) ; } if ( target . startsWith ( "?" ) ) target = baseRightMost + target ; return new URL ( base , target ) ; }


Input: Handles cases where the url param information is encoded into the base url as opposed to the target . <p > If the taget contains params ( i . e . ; xxxx ) information then the target params information is assumed to be correct and any base params information is ignored . If the base contains params information but the tareget does not then the params information is moved to the target allowing it to be correctly determined by the java . net . URL class . [CODESPLIT] private static URL fixEmbeddedParams ( URL base , String target ) throws MalformedURLException { // the target contains params information or the base doesn't then no // conversion necessary, return regular URL if ( target . indexOf ( ' ' ) >= 0 || base . toString ( ) . indexOf ( ' ' ) == - 1 ) { return new URL ( base , target ) ; } // get the base url and it params information String baseURL = base . toString ( ) ; int startParams = baseURL . indexOf ( ' ' ) ; String params = baseURL . substring ( startParams ) ; // if the target has a query string then put the params information // after // any path but before the query string, otherwise just append to the // path int startQS = target . indexOf ( ' ' ) ; if ( startQS >= 0 ) { target = target . substring ( 0 , startQS ) + params + target . substring ( startQS ) ; } else { target += params ; } return new URL ( base , target ) ; }


Input: Partitions of the hostname of the url by . [CODESPLIT] public static String [ ] getHostSegments ( URL url ) { String host = url . getHost ( ) ; // return whole hostname, if it is an ipv4 // TODO : handle ipv6 if ( IP_PATTERN . matcher ( host ) . matches ( ) ) return new String [ ] { host } ; return host . split ( "\\." ) ; }


Input: Returns the lowercased hostname for the url or null if the url is not well formed . [CODESPLIT] public static String getHost ( String url ) { try { return new URL ( url ) . getHost ( ) . toLowerCase ( Locale . ROOT ) ; } catch ( MalformedURLException e ) { return null ; } }


Input: Returns the page for the url . The page consists of the protocol host and path but does not include the query string . The host is lowercased but the path is not . [CODESPLIT] public static String getPage ( String url ) { try { // get the full url, and replace the query string with and empty // string url = url . toLowerCase ( Locale . ROOT ) ; String queryStr = new URL ( url ) . getQuery ( ) ; return ( queryStr != null ) ? url . replace ( "?" + queryStr , "" ) : url ; } catch ( MalformedURLException e ) { return null ; } }


Input: Return one or more Strings regardless of whether they are represented as a single String or a list in the config or an empty List if no value could be found for that key . [CODESPLIT] public static List < String > loadListFromConf ( String paramKey , Map stormConf ) { Object obj = stormConf . get ( paramKey ) ; List < String > list = new LinkedList <> ( ) ; if ( obj == null ) return list ; if ( obj instanceof PersistentVector ) { list . addAll ( ( PersistentVector ) obj ) ; } else { // single value? list . add ( obj . toString ( ) ) ; } return list ; }


Input: If the config consists of a single key config its values are used instead [CODESPLIT] public static Map extractConfigElement ( Map conf ) { if ( conf . size ( ) == 1 ) { Object confNode = conf . get ( "config" ) ; if ( confNode != null && confNode instanceof Map ) { conf = ( Map ) confNode ; } } return conf ; }


Input: Returns an instance of the protocol to use for a given URL [CODESPLIT] public synchronized Protocol getProtocol ( URL url ) { // get the protocol String protocol = url . getProtocol ( ) ; return cache . get ( protocol ) ; }


Input: Used for redirections or when discovering sitemap URLs . The custom key / values are added to the target metadata post - filtering . [CODESPLIT] protected void emitOutlink ( Tuple t , URL sURL , String newUrl , Metadata sourceMetadata , String ... customKeyVals ) { Outlink ol = filterOutlink ( sURL , newUrl , sourceMetadata , customKeyVals ) ; if ( ol == null ) return ; collector . emit ( com . digitalpebble . stormcrawler . Constants . StatusStreamName , t , new Values ( ol . getTargetURL ( ) , ol . getMetadata ( ) , Status . DISCOVERED ) ) ; }


Input: Generates a WARC info entry which can be stored at the beginning of each WARC file . [CODESPLIT] public static byte [ ] generateWARCInfo ( Map < String , String > fields ) { StringBuffer buffer = new StringBuffer ( ) ; buffer . append ( WARC_VERSION ) ; buffer . append ( CRLF ) ; buffer . append ( "WARC-Type: warcinfo" ) . append ( CRLF ) ; String mainID = UUID . randomUUID ( ) . toString ( ) ; // retrieve the date and filename from the map String date = fields . get ( "WARC-Date" ) ; buffer . append ( "WARC-Date: " ) . append ( date ) . append ( CRLF ) ; String filename = fields . get ( "WARC-Filename" ) ; buffer . append ( "WARC-Filename: " ) . append ( filename ) . append ( CRLF ) ; buffer . append ( "WARC-Record-ID" ) . append ( ": " ) . append ( "<urn:uuid:" ) . append ( mainID ) . append ( ">" ) . append ( CRLF ) ; buffer . append ( "Content-Type" ) . append ( ": " ) . append ( "application/warc-fields" ) . append ( CRLF ) ; StringBuilder fieldsBuffer = new StringBuilder ( ) ; // add WARC fields // http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf Iterator < Entry < String , String > > iter = fields . entrySet ( ) . iterator ( ) ; while ( iter . hasNext ( ) ) { Entry < String , String > entry = iter . next ( ) ; String key = entry . getKey ( ) ; if ( key . startsWith ( "WARC-" ) ) continue ; fieldsBuffer . append ( key ) . append ( ": " ) . append ( entry . getValue ( ) ) . append ( CRLF ) ; } buffer . append ( "Content-Length" ) . append ( ": " ) . append ( fieldsBuffer . toString ( ) . getBytes ( StandardCharsets . UTF_8 ) . length ) . append ( CRLF ) ; buffer . append ( CRLF ) ; buffer . append ( fieldsBuffer . toString ( ) ) ; buffer . append ( CRLF ) ; buffer . append ( CRLF ) ; return buffer . toString ( ) . getBytes ( StandardCharsets . UTF_8 ) ; }


Input:
Modify verbatim HTTP response headers : remove or replace headers <code > Content - Length< / code > <code > Content - Encoding< / code > and <code > Transfer - Encoding< / code > which may confuse WARC readers . Ensure that the header end with a single empty line ( <code > \ r \ n \ r \ n< / code > ) . [CODESPLIT] public static String fixHttpHeaders ( String headers , int contentLength ) { int start = 0 , lineEnd = 0 , last = 0 , trailingCrLf = 0 ; StringBuilder replace = new StringBuilder ( ) ; while ( start < headers . length ( ) ) { lineEnd = headers . indexOf ( CRLF , start ) ; trailingCrLf = 1 ; if ( lineEnd == - 1 ) { lineEnd = headers . length ( ) ; trailingCrLf = 0 ; } int colonPos = - 1 ; for ( int i = start ; i < lineEnd ; i ++ ) { if ( headers . charAt ( i ) == ' ' ) { colonPos = i ; break ; } } if ( colonPos == - 1 ) { boolean valid = true ; if ( start == 0 ) { // status line (without colon) } else if ( ( lineEnd + 4 ) == headers . length ( ) && headers . endsWith ( CRLF + CRLF ) ) { // ok, trailing empty line trailingCrLf = 2 ; } else if ( start == lineEnd ) { // skip/remove empty line valid = false ; } else { LOG . warn ( "Invalid header line: {}" , headers . substring ( start , lineEnd ) ) ; valid = false ; } if ( ! valid ) { if ( last < start ) { replace . append ( headers . substring ( last , start ) ) ; } last = lineEnd + 2 * trailingCrLf ; } start = lineEnd + 2 * trailingCrLf ; /*
                 * skip over invalid header line, no further check for
                 * problematic headers required
                 */ continue ; } String name = headers . substring ( start , colonPos ) ; if ( PROBLEMATIC_HEADERS . matcher ( name ) . matches ( ) ) { boolean needsFix = true ; if ( name . equalsIgnoreCase ( "content-length" ) ) { String value = headers . substring ( colonPos + 1 , lineEnd ) . trim ( ) ; try { int l = Integer . parseInt ( value ) ; if ( l == contentLength ) { needsFix = false ; } } catch ( NumberFormatException e ) { // needs to be fixed } } if ( needsFix ) { if ( last < start ) { replace . append ( headers . substring ( last , start ) ) ; } last = lineEnd + 2 * trailingCrLf ; replace . append ( X_HIDE_HEADER ) . append ( headers . substring ( start , lineEnd + 2 * trailingCrLf ) ) ; if ( trailingCrLf == 0 ) { replace . append ( CRLF ) ; trailingCrLf = 1 ; } if ( name . equalsIgnoreCase ( "content-length" ) ) { // add effective uncompressed and unchunked length of // content replace . append ( "Content-Length" ) . append ( ": " ) . append ( contentLength ) . append ( CRLF ) ; } } } start = lineEnd + 2 * trailingCrLf ; } if ( last > 0 || trailingCrLf != 2 ) { if ( last < headers . length ( ) ) { // append trailing headers replace . append ( headers . substring ( last ) ) ; } while ( trailingCrLf < 2 ) { replace . append ( CRLF ) ; trailingCrLf ++ ; } return replace . toString ( ) ; } return headers ; }


Input: Get the actual fetch time from metadata and format it as required by the WARC - Date field . If no fetch time is found in metadata ( key { [CODESPLIT] protected static String getCaptureTime ( Metadata metadata ) { String captureTimeMillis = metadata . getFirstValue ( REQUEST_TIME_KEY ) ; Instant capturedAt = Instant . now ( ) ; if ( captureTimeMillis != null ) { try { long millis = Long . parseLong ( captureTimeMillis ) ; capturedAt = Instant . ofEpochMilli ( millis ) ; } catch ( NumberFormatException | DateTimeException e ) { LOG . warn ( "Failed to parse capture time:" , e ) ; } } return WARC_DF . format ( capturedAt ) ; }


Input: Returns a normalised value of the content attribute for the refresh tag [CODESPLIT] public static String extractRefreshURL ( String value ) { if ( StringUtils . isBlank ( value ) ) return null ; // 0;URL=http://www.apollocolors.com/site try { if ( matcher . reset ( value ) . matches ( ) ) { return matcher . group ( 1 ) ; } } catch ( Exception e ) { } return null ; }


Input: Determine which metadata should be transfered to an outlink . Adds additional metadata like the URL path . [CODESPLIT] public Metadata getMetaForOutlink ( String targetURL , String sourceURL , Metadata parentMD ) { Metadata md = _filter ( parentMD , mdToTransfer ) ; // keep the path? if ( trackPath ) { md . addValue ( urlPathKeyName , sourceURL ) ; } // track depth if ( trackDepth ) { String existingDepth = md . getFirstValue ( depthKeyName ) ; int depth ; try { depth = Integer . parseInt ( existingDepth ) ; } catch ( Exception e ) { depth = 0 ; } md . setValue ( depthKeyName , Integer . toString ( ++ depth ) ) ; } return md ; }


Input: Determine which metadata should be persisted for a given document including those which are not necessarily transferred to the outlinks [CODESPLIT] public Metadata filter ( Metadata metadata ) { Metadata filtered_md = _filter ( metadata , mdToTransfer ) ; // add the features that are only persisted but // not transfered like __redirTo_ filtered_md . putAll ( _filter ( metadata , mdToPersistOnly ) ) ; return filtered_md ; }


Input: Populates a List of Rules off of JsonNode . [CODESPLIT] private List < RegexRule > readRules ( ArrayNode rulesList ) { List < RegexRule > rules = new ArrayList <> ( ) ; for ( JsonNode urlFilterNode : rulesList ) { try { RegexRule rule = createRule ( urlFilterNode . asText ( ) ) ; if ( rule != null ) { rules . add ( rule ) ; } } catch ( IOException e ) { LOG . error ( "There was an error reading regex filter {}" , urlFilterNode . asText ( ) , e ) ; } } return rules ; }


Input: / * -------------------------- * <implementation : URLFilter > * -------------------------- [CODESPLIT] @ Override public String filter ( URL pageUrl , Metadata sourceMetadata , String url ) { for ( RegexRule rule : rules ) { if ( rule . match ( url ) ) { return rule . accept ( ) ? url : null ; } } return null ; }


Input: Add the key value to the metadata object for a given URL * [CODESPLIT] public void put ( String URL , String key , String value ) { get ( URL ) . getMetadata ( ) . addValue ( key , value ) ; }


Input: Add a new URL [CODESPLIT] public static void add ( String url , Metadata md , Date nextFetch ) { LOG . debug ( "Adding {} with md {} and nextFetch {}" , url , md , nextFetch ) ; ScheduledURL tuple = new ScheduledURL ( url , md , nextFetch ) ; synchronized ( queue ) { queue . add ( tuple ) ; } }


Input: Returns a normalised doc ID based on the URL of a document * [CODESPLIT] public static String getID ( String url ) { // the document needs an ID // see // http://docs.aws.amazon.com/cloudsearch/latest/developerguide/preparing-data.html#creating-document-batches // A unique ID for the document. A document ID can contain any // letter or number and the following characters: _ - = # ; : / ? @ // &. Document IDs must be at least 1 and no more than 128 // characters long. byte [ ] dig = digester . digest ( url . getBytes ( StandardCharsets . UTF_8 ) ) ; String ID = Hex . encodeHexString ( dig ) ; // is that even possible? if ( ID . length ( ) > 128 ) { throw new RuntimeException ( "ID larger than max 128 chars" ) ; } return ID ; }


Input: Remove the non - cloudSearch - legal characters . Note that this might convert two fields to the same name . [CODESPLIT] public static String cleanFieldName ( String name ) { String lowercase = name . toLowerCase ( ) ; lowercase = lowercase . replaceAll ( "[^a-z_0-9]" , "_" ) ; if ( lowercase . length ( ) < 3 || lowercase . length ( ) > 64 ) throw new RuntimeException ( "Field name must be between 3 and 64 chars : " + lowercase ) ; if ( lowercase . equals ( "score" ) ) throw new RuntimeException ( "Field name must be score" ) ; return lowercase ; }


Input: Creates a connection with a default listener . The values for bolt type are [ indexer status metrics ] [CODESPLIT] public static ElasticSearchConnection getConnection ( Map stormConf , String boltType ) { BulkProcessor . Listener listener = new BulkProcessor . Listener ( ) { @ Override public void afterBulk ( long arg0 , BulkRequest arg1 , BulkResponse arg2 ) { } @ Override public void afterBulk ( long arg0 , BulkRequest arg1 , Throwable arg2 ) { } @ Override public void beforeBulk ( long arg0 , BulkRequest arg1 ) { } } ; return getConnection ( stormConf , boltType , listener ) ; }


Input: Identifies the charset of a document based on the following logic : guess from the ByteOrderMark - else if the same charset is specified in the http headers and the html metadata then use it - otherwise use ICU s charset detector to make an educated guess and if that fails too returns UTF - 8 . [CODESPLIT] public static String getCharset ( Metadata metadata , byte [ ] content , int maxLengthCharsetDetection ) { // let's look at the BOM first String BOMCharset = getCharsetFromBOM ( content ) ; if ( BOMCharset != null ) { return BOMCharset ; } // then look at what we get from HTTP headers and HTML content String httpCharset = getCharsetFromHTTP ( metadata ) ; String htmlCharset = getCharsetFromMeta ( content , maxLengthCharsetDetection ) ; // both exist and agree if ( httpCharset != null && htmlCharset != null && httpCharset . equalsIgnoreCase ( htmlCharset ) ) { return httpCharset ; } // let's guess from the text - using a hint or not String hintCharset = null ; if ( httpCharset != null && htmlCharset == null ) { hintCharset = httpCharset ; } else if ( httpCharset == null && htmlCharset != null ) { hintCharset = htmlCharset ; } String textCharset = getCharsetFromText ( content , hintCharset , maxLengthCharsetDetection ) ; if ( textCharset != null ) { return textCharset ; } // return the default charset return DEFAULT_CHARSET . name ( ) ; }


Input: Detects any BOMs and returns the corresponding charset [CODESPLIT] private static String getCharsetFromBOM ( final byte [ ] byteData ) { BOMInputStream bomIn = new BOMInputStream ( new ByteArrayInputStream ( byteData ) ) ; try { ByteOrderMark bom = bomIn . getBOM ( ) ; if ( bom != null ) { return bom . getCharsetName ( ) ; } } catch ( IOException e ) { return null ; } return null ; }


Input: Use a third party library as last resort to guess the charset from the bytes . [CODESPLIT] private static String getCharsetFromText ( byte [ ] content , String declaredCharset , int maxLengthCharsetDetection ) { String charset = null ; // filter HTML tags CharsetDetector charsetDetector = new CharsetDetector ( ) ; charsetDetector . enableInputFilter ( true ) ; // give it a hint if ( declaredCharset != null ) charsetDetector . setDeclaredEncoding ( declaredCharset ) ; // trim the content of the text for the detection byte [ ] subContent = content ; if ( maxLengthCharsetDetection != - 1 && content . length > maxLengthCharsetDetection ) { subContent = Arrays . copyOfRange ( content , 0 , maxLengthCharsetDetection ) ; } charsetDetector . setText ( subContent ) ; try { CharsetMatch charsetMatch = charsetDetector . detect ( ) ; charset = validateCharset ( charsetMatch . getName ( ) ) ; } catch ( Exception e ) { charset = null ; } return charset ; }


Input: Attempt to find a META tag in the HTML that hints at the character set used to write the document . [CODESPLIT] private static String getCharsetFromMeta ( byte buffer [ ] , int maxlength ) { // convert to UTF-8 String -- which hopefully will not mess up the // characters we're interested in... int len = buffer . length ; if ( maxlength > 0 && maxlength < len ) { len = maxlength ; } String html = new String ( buffer , 0 , len , DEFAULT_CHARSET ) ; Document doc = Parser . htmlParser ( ) . parseInput ( html , "dummy" ) ; // look for <meta http-equiv="Content-Type" // content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312"> Elements metaElements = doc . select ( "meta[http-equiv=content-type], meta[charset]" ) ; String foundCharset = null ; for ( Element meta : metaElements ) { if ( meta . hasAttr ( "http-equiv" ) ) foundCharset = getCharsetFromContentType ( meta . attr ( "content" ) ) ; if ( foundCharset == null && meta . hasAttr ( "charset" ) ) foundCharset = meta . attr ( "charset" ) ; if ( foundCharset != null ) return foundCharset ; } return foundCharset ; }


Input: Examines the first bytes of the content for a clue of whether this document is a sitemap based on namespaces . Works for XML and non - compressed documents only . [CODESPLIT] private final boolean sniff ( byte [ ] content ) { byte [ ] beginning = content ; if ( content . length > maxOffsetGuess && maxOffsetGuess > 0 ) { beginning = Arrays . copyOfRange ( content , 0 , maxOffsetGuess ) ; } int position = Bytes . indexOf ( beginning , clue ) ; if ( position != - 1 ) { return true ; } return false ; }


Input: Returns the host domain IP of a URL so that it can be partitioned for politeness depending on the value of the config <i > partition . url . mode< / i > . [CODESPLIT] public String getPartition ( String url , Metadata metadata ) { String partitionKey = null ; String host = "" ; // IP in metadata? if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_IP ) ) { String ip_provided = metadata . getFirstValue ( "ip" ) ; if ( StringUtils . isNotBlank ( ip_provided ) ) { partitionKey = ip_provided ; } } if ( partitionKey == null ) { URL u ; try { u = new URL ( url ) ; host = u . getHost ( ) ; } catch ( MalformedURLException e1 ) { LOG . warn ( "Invalid URL: {}" , url ) ; return null ; } } // partition by hostname if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_HOST ) ) partitionKey = host ; // partition by domain : needs fixing else if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_DOMAIN ) ) { partitionKey = PaidLevelDomain . getPLD ( host ) ; } // partition by IP if ( mode . equalsIgnoreCase ( Constants . PARTITION_MODE_IP ) && partitionKey == null ) { try { long start = System . currentTimeMillis ( ) ; final InetAddress addr = InetAddress . getByName ( host ) ; partitionKey = addr . getHostAddress ( ) ; long end = System . currentTimeMillis ( ) ; LOG . debug ( "Resolved IP {} in {} msec for : {}" , partitionKey , end - start , url ) ; } catch ( final Exception e ) { LOG . warn ( "Unable to resolve IP for: {}" , host ) ; return null ; } } LOG . debug ( "Partition Key for: {} > {}" , url , partitionKey ) ; return partitionKey ; }


Input: Set the value for a given key . The value can be null . [CODESPLIT] public void setValue ( String key , String value ) { md . put ( key , new String [ ] { value } ) ; }


Input: Returns the first non empty value found for the keys or null if none found . [CODESPLIT] public static String getFirstValue ( Metadata md , String ... keys ) { for ( String key : keys ) { String val = md . getFirstValue ( key ) ; if ( StringUtils . isBlank ( val ) ) continue ; return val ; } return null ; }


Input: Returns the first available driver * [CODESPLIT] private final RemoteWebDriver getDriver ( ) { try { return drivers . take ( ) ; } catch ( InterruptedException e ) { Thread . currentThread ( ) . interrupt ( ) ; } return null ; }


Input: Called by extensions of this class * [CODESPLIT] protected static void main ( AbstractHttpProtocol protocol , String args [ ] ) throws Exception { Config conf = new Config ( ) ; // loads the default configuration file Map defaultSCConfig = Utils . findAndReadConfigFile ( "crawler-default.yaml" , false ) ; conf . putAll ( ConfUtils . extractConfigElement ( defaultSCConfig ) ) ; Options options = new Options ( ) ; options . addOption ( "c" , true , "configuration file" ) ; CommandLineParser parser = new DefaultParser ( ) ; CommandLine cmd = parser . parse ( options , args ) ; if ( cmd . hasOption ( "c" ) ) { String confFile = cmd . getOptionValue ( "c" ) ; ConfUtils . loadConf ( confFile , conf ) ; } protocol . configure ( conf ) ; Set < Runnable > threads = new HashSet <> ( ) ; class Fetchable implements Runnable { String url ; Metadata md ; Fetchable ( String line ) { StringTabScheme scheme = new StringTabScheme ( ) ; List < Object > tuple = scheme . deserialize ( ByteBuffer . wrap ( line . getBytes ( StandardCharsets . UTF_8 ) ) ) ; this . url = ( String ) tuple . get ( 0 ) ; this . md = ( Metadata ) tuple . get ( 1 ) ; } public void run ( ) { StringBuilder stringB = new StringBuilder ( ) ; stringB . append ( url ) . append ( "\n" ) ; if ( ! protocol . skipRobots ) { BaseRobotRules rules = protocol . getRobotRules ( url ) ; stringB . append ( "robots allowed: " ) . append ( rules . isAllowed ( url ) ) . append ( "\n" ) ; if ( rules instanceof RobotRules ) { stringB . append ( "robots requests: " ) . append ( ( ( RobotRules ) rules ) . getContentLengthFetched ( ) . length ) . append ( "\n" ) ; } stringB . append ( "sitemaps identified: " ) . append ( rules . getSitemaps ( ) . size ( ) ) . append ( "\n" ) ; } long start = System . currentTimeMillis ( ) ; ProtocolResponse response ; try { response = protocol . getProtocolOutput ( url , md ) ; stringB . append ( response . getMetadata ( ) ) . append ( "\n" ) ; stringB . append ( "status code: " ) . append ( response . getStatusCode ( ) ) . append ( "\n" ) ; stringB . append ( "content length: " ) . append ( response . getContent ( ) . length ) . append ( "\n" ) ; long timeFetching = System . currentTimeMillis ( ) - start ; stringB . append ( "fetched in : " ) . append ( timeFetching ) . append ( " msec" ) ; System . out . println ( stringB ) ; } catch ( Exception e ) { e . printStackTrace ( ) ; } finally { threads . remove ( this ) ; } } } for ( String arg : cmd . getArgs ( ) ) { Fetchable p = new Fetchable ( arg ) ; threads . add ( p ) ; new Thread ( p ) . start ( ) ; } while ( threads . size ( ) > 0 ) { Thread . sleep ( 1000 ) ; } protocol . cleanup ( ) ; System . exit ( 0 ) ; }


Input: Get a list of cookies based on the cookies string taken from response header and the target url . [CODESPLIT] public static List < Cookie > getCookies ( String [ ] cookiesStrings , URL targetURL ) { ArrayList < Cookie > list = new ArrayList < Cookie > ( ) ; for ( String cs : cookiesStrings ) { String name = null ; String value = null ; String expires = null ; String domain = null ; String path = null ; boolean secure = false ; String [ ] tokens = cs . split ( ";" ) ; int equals = tokens [ 0 ] . indexOf ( "=" ) ; name = tokens [ 0 ] . substring ( 0 , equals ) ; value = tokens [ 0 ] . substring ( equals + 1 ) ; for ( int i = 1 ; i < tokens . length ; i ++ ) { String ti = tokens [ i ] . trim ( ) ; if ( ti . equalsIgnoreCase ( "secure" ) ) secure = true ; if ( ti . toLowerCase ( ) . startsWith ( "path=" ) ) { path = ti . substring ( 5 ) ; } if ( ti . toLowerCase ( ) . startsWith ( "domain=" ) ) { domain = ti . substring ( 7 ) ; } if ( ti . toLowerCase ( ) . startsWith ( "expires=" ) ) { expires = ti . substring ( 8 ) ; } } BasicClientCookie cookie = new BasicClientCookie ( name , value ) ; // check domain if ( domain != null ) { cookie . setDomain ( domain ) ; if ( ! checkDomainMatchToUrl ( domain , targetURL . getHost ( ) ) ) continue ; } // check path if ( path != null ) { cookie . setPath ( path ) ; if ( ! path . equals ( "" ) && ! path . equals ( "/" ) && ! targetURL . getPath ( ) . startsWith ( path ) ) continue ; } // check secure if ( secure ) { cookie . setSecure ( secure ) ; if ( ! targetURL . getProtocol ( ) . equalsIgnoreCase ( "https" ) ) continue ; } // check expiration if ( expires != null ) { try { Date expirationDate = DATE_FORMAT . parse ( expires ) ; cookie . setExpiryDate ( expirationDate ) ; // check that it hasn't expired? if ( cookie . isExpired ( new Date ( ) ) ) continue ; cookie . setExpiryDate ( expirationDate ) ; } catch ( ParseException e ) { // ignore exceptions } } // attach additional infos to cookie list . add ( cookie ) ; } return list ; }


Input: Helper method to check if url matches a cookie domain . [CODESPLIT] public static boolean checkDomainMatchToUrl ( String cookieDomain , String urlHostName ) { try { if ( cookieDomain . startsWith ( "." ) ) { cookieDomain = cookieDomain . substring ( 1 ) ; } String [ ] domainTokens = cookieDomain . split ( "\\." ) ; String [ ] hostTokens = urlHostName . split ( "\\." ) ; int tokenDif = hostTokens . length - domainTokens . length ; if ( tokenDif < 0 ) { return false ; } for ( int i = domainTokens . length - 1 ; i >= 0 ; i -- ) { if ( ! domainTokens [ i ] . equalsIgnoreCase ( hostTokens [ i + tokenDif ] ) ) { return false ; } } return true ; } catch ( Exception e ) { return true ; } }


Input:
Compose unique key to store and access robot rules in cache for given URL [CODESPLIT] protected static String getCacheKey ( URL url ) { String protocol = url . getProtocol ( ) . toLowerCase ( Locale . ROOT ) ; String host = url . getHost ( ) . toLowerCase ( Locale . ROOT ) ; int port = url . getPort ( ) ; if ( port == - 1 ) { port = url . getDefaultPort ( ) ; } /*
         * Robot rules apply only to host, protocol, and port where robots.txt
         * is hosted (cf. NUTCH-1752). Consequently
         */ return protocol + ":" + host + ":" + port ; }


Input: Returns the robots rules from the cache or empty rules if not found [CODESPLIT] public BaseRobotRules getRobotRulesSetFromCache ( URL url ) { String cacheKey = getCacheKey ( url ) ; BaseRobotRules robotRules = CACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } return EMPTY_RULES ; }


Input: Get the rules from robots . txt which applies for the given { @code url } . Robot rules are cached for a unique combination of host protocol and port . If no rules are found in the cache a HTTP request is send to fetch {{ protocol : // host : port / robots . txt }} . The robots . txt is then parsed and the rules are cached to avoid re - fetching and re - parsing it again . [CODESPLIT] @ Override public BaseRobotRules getRobotRulesSet ( Protocol http , URL url ) { String cacheKey = getCacheKey ( url ) ; // check in the error cache first BaseRobotRules robotRules = ERRORCACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } // now try the proper cache robotRules = CACHE . getIfPresent ( cacheKey ) ; if ( robotRules != null ) { return robotRules ; } boolean cacheRule = true ; URL redir = null ; String keyredir = null ; LOG . debug ( "Cache miss {} for {}" , cacheKey , url ) ; List < Integer > bytesFetched = new LinkedList <> ( ) ; try { ProtocolResponse response = http . getProtocolOutput ( new URL ( url , "/robots.txt" ) . toString ( ) , Metadata . empty ) ; int code = response . getStatusCode ( ) ; bytesFetched . add ( response . getContent ( ) != null ? response . getContent ( ) . length : 0 ) ; // try one level of redirection ? if ( code == 301 || code == 302 || code == 307 || code == 308 ) { String redirection = response . getMetadata ( ) . getFirstValue ( HttpHeaders . LOCATION ) ; if ( StringUtils . isNotBlank ( redirection ) ) { if ( ! redirection . startsWith ( "http" ) ) { // RFC says it should be absolute, but apparently it // isn't redir = new URL ( url , redirection ) ; } else { redir = new URL ( redirection ) ; } // try from the cache keyredir = getCacheKey ( redir ) ; if ( cacheKey . equalsIgnoreCase ( keyredir ) ) { keyredir = null ; } else { RobotRules cachedRediRobotRules = CACHE . getIfPresent ( keyredir ) ; if ( cachedRediRobotRules != null ) { // cache also for the redirected host // but only if the robots.txt file is at the root if ( redir . getPath ( ) . equals ( "/robots.txt" ) ) { LOG . debug ( "Caching robots for {} under key {} in cache" , redir , keyredir ) ; CACHE . put ( keyredir , cachedRediRobotRules ) ; } return cachedRediRobotRules ; } } response = http . getProtocolOutput ( redir . toString ( ) , Metadata . empty ) ; code = response . getStatusCode ( ) ; bytesFetched . add ( response . getContent ( ) != null ? response . getContent ( ) . length : 0 ) ; } } if ( code == 200 ) // found rules: parse them { String ct = response . getMetadata ( ) . getFirstValue ( HttpHeaders . CONTENT_TYPE ) ; robotRules = parseRules ( url . toString ( ) , response . getContent ( ) , ct , agentNames ) ; } else if ( ( code == 403 ) && ( ! allowForbidden ) ) { robotRules = FORBID_ALL_RULES ; // use forbid all } else if ( code >= 500 ) { cacheRule = false ; robotRules = EMPTY_RULES ; } else robotRules = EMPTY_RULES ; // use default rules } catch ( Throwable t ) { LOG . info ( "Couldn't get robots.txt for {} : {}" , url , t . toString ( ) ) ; cacheRule = false ; robotRules = EMPTY_RULES ; } Cache < String , RobotRules > cacheToUse = CACHE ; String cacheName = "success" ; if ( ! cacheRule ) { cacheToUse = ERRORCACHE ; cacheName = "error" ; } RobotRules cached = new RobotRules ( robotRules ) ; LOG . debug ( "Caching robots for {} under key {} in cache {}" , url , cacheKey , cacheName ) ; cacheToUse . put ( cacheKey , cached ) ; // cache robot for redirections // get here only if the target has not been found in the cache if ( keyredir != null ) { // cache also for the redirected host // but only if the robots.txt file is at the root if ( redir . getPath ( ) . equals ( "/robots.txt" ) ) { LOG . debug ( "Caching robots for {} under key {} in cache {}" , redir , keyredir , cacheName ) ; cacheToUse . put ( keyredir , cached ) ; } } RobotRules live = new RobotRules ( robotRules ) ; live . setContentLengthFetched ( Ints . toArray ( bytesFetched ) ) ; return live ; }


Input: called by the parser bolts [CODESPLIT] public void extractMetaTags ( DocumentFragment doc ) throws XPathExpressionException { NodeList nodes = ( NodeList ) expression . evaluate ( doc , XPathConstants . NODESET ) ; if ( nodes == null ) return ; int numNodes = nodes . getLength ( ) ; for ( int i = 0 ; i < numNodes ; i ++ ) { Node n = ( Node ) nodes . item ( i ) ; // iterate on the attributes // and check that it has name=robots and content // whatever the case is boolean isRobots = false ; String content = null ; NamedNodeMap attrs = n . getAttributes ( ) ; for ( int att = 0 ; att < attrs . getLength ( ) ; att ++ ) { Node keyval = attrs . item ( att ) ; if ( "name" . equalsIgnoreCase ( keyval . getNodeName ( ) ) && "robots" . equalsIgnoreCase ( keyval . getNodeValue ( ) ) ) { isRobots = true ; continue ; } if ( "content" . equalsIgnoreCase ( keyval . getNodeName ( ) ) ) { content = keyval . getNodeValue ( ) ; continue ; } } if ( isRobots && content != null ) { // got a value - split it String [ ] vals = content . split ( " *, *" ) ; parseValues ( vals ) ; return ; } } }


Input: Extracts meta tags based on the value of the content attribute * [CODESPLIT] public void extractMetaTags ( String content ) { if ( content == null ) return ; String [ ] vals = content . split ( " *, *" ) ; parseValues ( vals ) ; }


Input: Adds a normalised representation of the directives in the metadata * [CODESPLIT] public void normaliseToMetadata ( Metadata metadata ) { metadata . setValue ( ROBOTS_NO_INDEX , Boolean . toString ( noIndex ) ) ; metadata . setValue ( ROBOTS_NO_CACHE , Boolean . toString ( noCache ) ) ; metadata . setValue ( ROBOTS_NO_FOLLOW , Boolean . toString ( noFollow ) ) ; }


Input: Must be called by extending classes to store and collect in one go [CODESPLIT] protected final void ack ( Tuple t , String url ) { // keep the URL in the cache if ( useCache ) { cache . put ( url , "" ) ; } _collector . ack ( t ) ; }


Input: Try the rules from the hostname domain name metadata and global scopes in this order . Returns true if the URL should be removed false otherwise . The value returns the value of the first matching rule be it positive or negative . [CODESPLIT] public boolean filter ( String url , Metadata metadata ) throws MalformedURLException { URL u = new URL ( url ) ; // first try the full hostname String hostname = u . getHost ( ) ; if ( checkScope ( hostNameRules . get ( hostname ) , u ) ) { return true ; } // then on the various components of the domain String [ ] domainParts = hostname . split ( "\\." ) ; String domain = null ; for ( int i = domainParts . length - 1 ; i >= 0 ; i -- ) { domain = domainParts [ i ] + ( domain == null ? "" : "." + domain ) ; if ( checkScope ( domainRules . get ( domain ) , u ) ) { return true ; } } // check on parent's URL metadata for ( MDScope scope : metadataRules ) { String [ ] vals = metadata . getValues ( scope . getKey ( ) ) ; if ( vals == null ) { continue ; } for ( String v : vals ) { if ( v . equalsIgnoreCase ( scope . getValue ( ) ) ) { FastURLFilter . LOG . debug ( "Filtering {} matching metadata {}:{}" , url , scope . getKey ( ) , scope . getValue ( ) ) ; if ( checkScope ( scope , u ) ) { return true ; } } } } if ( checkScope ( globalRules , u ) ) { return true ; } return false ; }


Input: This function does the replacements by iterating through all the regex patterns . It accepts a string url as input and returns the altered string . If the normalized url is an empty string the function will return null . [CODESPLIT] @ Override public String filter ( URL sourceUrl , Metadata sourceMetadata , String urlString ) { Iterator < Rule > i = rules . iterator ( ) ; while ( i . hasNext ( ) ) { Rule r = i . next ( ) ; Matcher matcher = r . pattern . matcher ( urlString ) ; urlString = matcher . replaceAll ( r . substitution ) ; } if ( urlString . equals ( "" ) ) { urlString = null ; } return urlString ; }


Input: Populates a List of Rules off of JsonNode . [CODESPLIT] private List < Rule > readRules ( ArrayNode rulesList ) { List < Rule > rules = new ArrayList <> ( ) ; for ( JsonNode regexNode : rulesList ) { if ( regexNode == null || regexNode . isNull ( ) ) { LOG . warn ( "bad config: 'regex' element is null" ) ; continue ; } JsonNode patternNode = regexNode . get ( "pattern" ) ; JsonNode substitutionNode = regexNode . get ( "substitution" ) ; String substitutionValue = "" ; if ( substitutionNode != null ) { substitutionValue = substitutionNode . asText ( ) ; } if ( patternNode != null && StringUtils . isNotBlank ( patternNode . asText ( ) ) ) { Rule rule = createRule ( patternNode . asText ( ) , substitutionValue ) ; if ( rule != null ) { rules . add ( rule ) ; } } } if ( rules . size ( ) == 0 ) { rules = EMPTY_RULES ; } return rules ; }


Input: Reads the configuration file and populates a List of Rules . [CODESPLIT] private List < Rule > readRules ( String rulesFile ) { try { InputStream regexStream = getClass ( ) . getClassLoader ( ) . getResourceAsStream ( rulesFile ) ; Reader reader = new InputStreamReader ( regexStream , StandardCharsets . UTF_8 ) ; return readConfiguration ( reader ) ; } catch ( Exception e ) { LOG . error ( "Error loading rules from file: {}" , e ) ; return EMPTY_RULES ; } }


Input: Basic filter to remove query parameters from urls so parameters that don t change the content of the page can be removed . An example would be a google analytics query parameter like utm_campaign which might have several different values for a url that points to the same content . This is also called when removing attributes where the value is a hash . [CODESPLIT] private String processQueryElements ( String urlToFilter ) { try { // Handle illegal characters by making a url first // this will clean illegal characters like | URL url = new URL ( urlToFilter ) ; String query = url . getQuery ( ) ; String path = url . getPath ( ) ; // check if the last element of the path contains parameters // if so convert them to query elements if ( path . contains ( ";" ) ) { String [ ] pathElements = path . split ( "/" ) ; String last = pathElements [ pathElements . length - 1 ] ; // replace last value by part without params int semicolon = last . indexOf ( ";" ) ; if ( semicolon != - 1 ) { pathElements [ pathElements . length - 1 ] = last . substring ( 0 , semicolon ) ; String params = last . substring ( semicolon + 1 ) . replaceAll ( ";" , "&" ) ; if ( query == null ) { query = params ; } else { query += "&" + params ; } // rebuild the path StringBuilder newPath = new StringBuilder ( ) ; for ( String p : pathElements ) { if ( StringUtils . isNotBlank ( p ) ) { newPath . append ( "/" ) . append ( p ) ; } } path = newPath . toString ( ) ; } } if ( StringUtils . isEmpty ( query ) ) { return urlToFilter ; } List < NameValuePair > pairs = URLEncodedUtils . parse ( query , StandardCharsets . UTF_8 ) ; Iterator < NameValuePair > pairsIterator = pairs . iterator ( ) ; while ( pairsIterator . hasNext ( ) ) { NameValuePair param = pairsIterator . next ( ) ; if ( queryElementsToRemove . contains ( param . getName ( ) ) ) { pairsIterator . remove ( ) ; } else if ( removeHashes && param . getValue ( ) != null ) { Matcher m = thirtytwobithash . matcher ( param . getValue ( ) ) ; if ( m . matches ( ) ) { pairsIterator . remove ( ) ; } } } StringBuilder newFile = new StringBuilder ( ) ; if ( StringUtils . isNotBlank ( path ) ) { newFile . append ( path ) ; } if ( ! pairs . isEmpty ( ) ) { Collections . sort ( pairs , comp ) ; String newQueryString = URLEncodedUtils . format ( pairs , StandardCharsets . UTF_8 ) ; newFile . append ( ' ' ) . append ( newQueryString ) ; } if ( url . getRef ( ) != null ) { newFile . append ( ' ' ) . append ( url . getRef ( ) ) ; } return new URL ( url . getProtocol ( ) , url . getHost ( ) , url . getPort ( ) , newFile . toString ( ) ) . toString ( ) ; } catch ( MalformedURLException e ) { LOG . warn ( "Invalid urlToFilter {}. {}" , urlToFilter , e ) ; return null ; } }


Input: A common error to find is a query string that starts with an & instead of a ? This will fix that error . So http : // foo . com&a = b will be changed to http : // foo . com?a = b . [CODESPLIT] private String unmangleQueryString ( String urlToFilter ) { int firstAmp = urlToFilter . indexOf ( ' ' ) ; if ( firstAmp > 0 ) { int firstQuestionMark = urlToFilter . indexOf ( ' ' ) ; if ( firstQuestionMark == - 1 ) { return urlToFilter . replaceFirst ( "&" , "?" ) ; } } return urlToFilter ; }


Input: Remove % encoding from path segment in URL for characters which should be unescaped according to <a href = https : // tools . ietf . org / html / rfc3986#section - 2 . 2 > RFC3986< / a > as well as non - standard implementations of percent encoding see <https : // en . wikipedia . org / wiki / Percent - encoding#Non - standard_implementations > . [CODESPLIT] private String unescapePath ( String path ) { Matcher matcher = illegalEscapePattern . matcher ( path ) ; StringBuilder sb = null ; int end = 0 ; while ( matcher . find ( ) ) { if ( sb == null ) { sb = new StringBuilder ( ) ; } // Append everything up to this group sb . append ( path . substring ( end , matcher . start ( ) ) ) ; String group = matcher . group ( 1 ) ; int letter = Integer . valueOf ( group , 16 ) ; sb . append ( ( char ) letter ) ; end = matcher . end ( ) ; } // we got a replacement if ( sb != null ) { // append whatever is left sb . append ( path . substring ( end ) ) ; path = sb . toString ( ) ; end = 0 ; } matcher = unescapeRulePattern . matcher ( path ) ; if ( ! matcher . find ( ) ) { return path ; } sb = new StringBuilder ( ) ; // Traverse over all encoded groups do { // Append everything up to this group sb . append ( path . substring ( end , matcher . start ( ) ) ) ; // Get the integer representation of this hexadecimal encoded // character int letter = Integer . valueOf ( matcher . group ( 1 ) , 16 ) ; if ( letter < 128 && unescapedCharacters [ letter ] ) { // character should be unescaped in URLs sb . append ( ( char ) letter ) ; } else { // Append the whole sequence as uppercase sb . append ( matcher . group ( ) . toUpperCase ( Locale . ROOT ) ) ; } end = matcher . end ( ) ; } while ( matcher . find ( ) ) ; // Append the rest if there's anything left sb . append ( path . substring ( end ) ) ; return sb . toString ( ) ; }


Input: Convert path segment of URL from Unicode to UTF - 8 and escape all characters which should be escaped according to <a href = https : // tools . ietf . org / html / rfc3986#section - 2 . 2 > RFC3986< / a > .. [CODESPLIT] private String escapePath ( String path ) { StringBuilder sb = new StringBuilder ( path . length ( ) ) ; // Traverse over all bytes in this URL for ( byte b : path . getBytes ( utf8 ) ) { // Is this a control character? if ( b < 33 || b == 91 || b == 92 || b == 93 || b == 124 ) { // Start escape sequence sb . append ( ' ' ) ; // Get this byte's hexadecimal representation String hex = Integer . toHexString ( b & 0xFF ) . toUpperCase ( Locale . ROOT ) ; // Do we need to prepend a zero? if ( hex . length ( ) % 2 != 0 ) { sb . append ( ' ' ) ; sb . append ( hex ) ; } else { // No, append this hexadecimal representation sb . append ( hex ) ; } } else { // No, just append this character as-is sb . append ( ( char ) b ) ; } } return sb . toString ( ) ; }


Input: Returns an S3 client given the configuration * [CODESPLIT] public static AmazonS3Client getS3Client ( Map conf ) { AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain ( ) ; AWSCredentials credentials = provider . getCredentials ( ) ; ClientConfiguration config = new ClientConfiguration ( ) ; AmazonS3Client client = new AmazonS3Client ( credentials , config ) ; String regionName = ConfUtils . getString ( conf , REGION ) ; if ( StringUtils . isNotBlank ( regionName ) ) { client . setRegion ( RegionUtils . getRegion ( regionName ) ) ; } String endpoint = ConfUtils . getString ( conf , ENDPOINT ) ; if ( StringUtils . isNotBlank ( endpoint ) ) { client . setEndpoint ( endpoint ) ; } return client ; }


Input: Loads and configure the NavigationFilters based on the storm config if there is one otherwise returns an emptyNavigationFilters . [CODESPLIT] @ SuppressWarnings ( "rawtypes" ) public static NavigationFilters fromConf ( Map stormConf ) { String configfile = ConfUtils . getString ( stormConf , "navigationfilters.config.file" ) ; if ( StringUtils . isNotBlank ( configfile ) ) { try { return new NavigationFilters ( stormConf , configfile ) ; } catch ( IOException e ) { String message = "Exception caught while loading the NavigationFilters from " + configfile ; LOG . error ( message ) ; throw new RuntimeException ( message , e ) ; } } return NavigationFilters . emptyNavigationFilters ; }


Input: Add an additional record format at given position [CODESPLIT] public GzipHdfsBolt addRecordFormat ( RecordFormat format , int position ) { MultipleRecordFormat formats ; if ( this . format == null ) { formats = new MultipleRecordFormat ( format ) ; this . format = formats ; } else { if ( this . format instanceof MultipleRecordFormat ) { formats = ( MultipleRecordFormat ) this . format ; } else { formats = new MultipleRecordFormat ( this . format ) ; this . format = formats ; } formats . addFormat ( new GzippedRecordFormat ( format ) , position ) ; } return this ; }


Input: Returns the amount of time to wait if the backend was queried too recently and needs throttling or - 1 if the backend can be queried straight away . [CODESPLIT] private long throttleQueries ( ) { if ( timeLastQuerySent != 0 ) { // check that we allowed some time between queries long difference = System . currentTimeMillis ( ) - timeLastQuerySent ; if ( difference < minDelayBetweenQueries ) { return minDelayBetweenQueries - difference ; } } return - 1 ; }


Input: Indicates whether enough time has elapsed since receiving the results of the previous query so that a new one can be sent even if the buffer is not empty . Applies to asynchronous clients only . [CODESPLIT] private boolean triggerQueries ( ) { if ( timeLastQueryReceived != 0 && maxDelayBetweenQueries > 0 ) { // check that we allowed some time between queries long difference = System . currentTimeMillis ( ) - timeLastQueryReceived ; if ( difference > maxDelayBetweenQueries ) { return true ; } } return false ; }


Input: Loads and configure the ParseFilters based on the storm config if there is one otherwise returns an emptyParseFilter . [CODESPLIT] @ SuppressWarnings ( "rawtypes" ) public static ParseFilters fromConf ( Map stormConf ) { String parseconfigfile = ConfUtils . getString ( stormConf , "parsefilters.config.file" ) ; if ( StringUtils . isNotBlank ( parseconfigfile ) ) { try { return new ParseFilters ( stormConf , parseconfigfile ) ; } catch ( IOException e ) { String message = "Exception caught while loading the ParseFilters from " + parseconfigfile ; LOG . error ( message ) ; throw new RuntimeException ( message , e ) ; } } return ParseFilters . emptyParseFilter ; }


Input: Append a node to the current container . [CODESPLIT] protected void append ( Node newNode ) throws org . xml . sax . SAXException { Node currentNode = m_currentNode ; if ( null != currentNode ) { currentNode . appendChild ( newNode ) ; // System.out.println(newNode.getNodeName()); } else if ( null != m_docFrag ) { m_docFrag . appendChild ( newNode ) ; } else { boolean ok = true ; short type = newNode . getNodeType ( ) ; if ( type == Node . TEXT_NODE ) { String data = newNode . getNodeValue ( ) ; if ( ( null != data ) && ( data . trim ( ) . length ( ) > 0 ) ) { throw new org . xml . sax . SAXException ( "Warning: can't output text before document element!  Ignoring..." ) ; } ok = false ; } else if ( type == Node . ELEMENT_NODE ) { if ( m_doc . getDocumentElement ( ) != null ) { throw new org . xml . sax . SAXException ( "Can't have more than one root on a DOM!" ) ; } } if ( ok ) { m_doc . appendChild ( newNode ) ; } } }


Input: Receive notification of ignorable whitespace in element content . [CODESPLIT] @ Override public void ignorableWhitespace ( char ch [ ] , int start , int length ) throws org . xml . sax . SAXException { if ( isOutsideDocElem ( ) ) { return ; // avoid DOM006 Hierarchy request error } String s = new String ( ch , start , length ) ; append ( m_doc . createTextNode ( s ) ) ; }


Input: Receive notification of a processing instruction . [CODESPLIT] @ Override public void processingInstruction ( String target , String data ) throws org . xml . sax . SAXException { append ( m_doc . createProcessingInstruction ( target , data ) ) ; }


Input: Report an XML comment anywhere in the document . [CODESPLIT] @ Override public void comment ( char ch [ ] , int start , int length ) throws org . xml . sax . SAXException { // tagsoup sometimes submits invalid values here if ( ch == null || start < 0 || length >= ( ch . length - start ) || length < 0 ) { return ; } append ( m_doc . createComment ( new String ( ch , start , length ) ) ) ; }


Input: Receive notification of cdata . [CODESPLIT] public void cdata ( char ch [ ] , int start , int length ) { if ( isOutsideDocElem ( ) && XMLCharacterRecognizer . isWhiteSpace ( ch , start , length ) ) { return ; // avoid DOM006 Hierarchy request error } String s = new String ( ch , start , length ) ; // XXX ab@apache.org: modified from the original, to accomodate TagSoup. Node n = m_currentNode . getLastChild ( ) ; if ( n instanceof CDATASection ) { ( ( CDATASection ) n ) . appendData ( s ) ; } else if ( n instanceof Comment ) { ( ( Comment ) n ) . appendData ( s ) ; } }


Input: Report the start of DTD declarations if any . [CODESPLIT] @ Override public void startDTD ( String name , String publicId , String systemId ) throws org . xml . sax . SAXException { // Do nothing for now. }


Input:
Begin the scope of a prefix - URI Namespace mapping . [CODESPLIT] @ Override public void startPrefixMapping ( String prefix , String uri ) throws org . xml . sax . SAXException { /*
         * // Not sure if this is needed or wanted // Also, it fails in the
         * stree. if((null != m_currentNode) && (m_currentNode.getNodeType() ==
         * Node.ELEMENT_NODE)) { String qname; if(((null != prefix) &&
         * (prefix.length() == 0)) || (null == prefix)) qname = "xmlns"; else
         * qname = "xmlns:"+prefix;
         *
         * Element elem = (Element)m_currentNode; String val =
         * elem.getAttribute(qname); // Obsolete, should be DOM2...? if(val ==
         * null) { elem.setAttributeNS("http://www.w3.org/XML/1998/namespace",
         * qname, uri); } }
         */ }


Input:
Set the { [CODESPLIT] public void setConf ( Config conf ) { // Grab the agent names we advertise to robots files. String agentName = ConfUtils . getString ( conf , "http.agent.name" ) ; if ( null == agentName ) { throw new RuntimeException ( "Agent name not configured!" ) ; } String configuredAgentNames = ConfUtils . getString ( conf , "http.robots.agents" , "" ) ; StringTokenizer tok = new StringTokenizer ( configuredAgentNames , "," ) ; ArrayList < String > agents = new ArrayList <> ( ) ; while ( tok . hasMoreTokens ( ) ) { agents . add ( tok . nextToken ( ) . trim ( ) ) ; } /**
         * If there are no agents for robots-parsing, use the default
         * agent-string. If both are present, our agent-string should be the
         * first one we advertise to robots-parsing.
         */ if ( agents . isEmpty ( ) ) { LOG . info ( "No agents listed in 'http.robots.agents' property! Using http.agent.name [{}]" , agentName ) ; this . agentNames = agentName ; } else { int index = 0 ; if ( ( agents . get ( 0 ) ) . equalsIgnoreCase ( agentName ) ) { index ++ ; } else { LOG . info ( "Agent we advertise ({}) not listed first in 'http.robots.agents' property!" , agentName ) ; } StringBuilder combinedAgentsString = new StringBuilder ( agentName ) ; // append all the agents from the http.robots.agents property for ( ; index < agents . size ( ) ; index ++ ) { combinedAgentsString . append ( ", " ) . append ( agents . get ( index ) ) ; } this . agentNames = combinedAgentsString . toString ( ) ; } String spec = ConfUtils . getString ( conf , cacheConfigParamName , "maximumSize=10000,expireAfterWrite=6h" ) ; CACHE = CacheBuilder . from ( spec ) . build ( ) ; spec = ConfUtils . getString ( conf , errorcacheConfigParamName , "maximumSize=10000,expireAfterWrite=1h" ) ; ERRORCACHE = CacheBuilder . from ( spec ) . build ( ) ; }


Input: Parses the robots content using the { @link SimpleRobotRulesParser } from crawler commons [CODESPLIT] public BaseRobotRules parseRules ( String url , byte [ ] content , String contentType , String robotName ) { return robotParser . parseContent ( url , content , contentType , robotName ) ; }


Input: Determine whether a document should be indexed based on the presence of a given key / value or the RobotsTags . ROBOTS_NO_INDEX directive . [CODESPLIT] protected boolean filterDocument ( Metadata meta ) { String noindexVal = meta . getFirstValue ( RobotsTags . ROBOTS_NO_INDEX ) ; if ( "true" . equalsIgnoreCase ( noindexVal ) ) return false ; if ( filterKeyValue == null ) return true ; String [ ] values = meta . getValues ( filterKeyValue [ 0 ] ) ; // key not found if ( values == null ) return false ; return ArrayUtils . contains ( values , filterKeyValue [ 1 ] ) ; }


Input: Returns a mapping field name / values for the metadata to index * [CODESPLIT] protected Map < String , String [ ] > filterMetadata ( Metadata meta ) { Pattern indexValuePattern = Pattern . compile ( "\\[(\\d+)\\]" ) ; Map < String , String [ ] > fieldVals = new HashMap <> ( ) ; Iterator < Entry < String , String > > iter = metadata2field . entrySet ( ) . iterator ( ) ; while ( iter . hasNext ( ) ) { Entry < String , String > entry = iter . next ( ) ; // check whether we want a specific value or all of them? int index = - 1 ; String key = entry . getKey ( ) ; Matcher match = indexValuePattern . matcher ( key ) ; if ( match . find ( ) ) { index = Integer . parseInt ( match . group ( 1 ) ) ; key = key . substring ( 0 , match . start ( ) ) ; } String [ ] values = meta . getValues ( key ) ; // not found if ( values == null || values . length == 0 ) continue ; // want a value index that it outside the range given if ( index >= values . length ) continue ; // store all values available if ( index == - 1 ) fieldVals . put ( entry . getValue ( ) , values ) ; // or only the one we want else fieldVals . put ( entry . getValue ( ) , new String [ ] { values [ index ] } ) ; } return fieldVals ; }


Input: Returns the value to be used as the URL for indexing purposes if present the canonical value is used instead [CODESPLIT] protected String valueForURL ( Tuple tuple ) { String url = tuple . getStringByField ( "url" ) ; Metadata metadata = ( Metadata ) tuple . getValueByField ( "metadata" ) ; // functionality deactivated if ( StringUtils . isBlank ( canonicalMetadataParamName ) ) { return url ; } String canonicalValue = metadata . getFirstValue ( canonicalMetadataName ) ; // no value found? if ( StringUtils . isBlank ( canonicalValue ) ) { return url ; } try { URL sURL = new URL ( url ) ; URL canonical = URLUtil . resolveURL ( sURL , canonicalValue ) ; String sDomain = PaidLevelDomain . getPLD ( sURL . getHost ( ) ) ; String canonicalDomain = PaidLevelDomain . getPLD ( canonical . getHost ( ) ) ; // check that the domain is the same if ( sDomain . equalsIgnoreCase ( canonicalDomain ) ) { return canonical . toExternalForm ( ) ; } else { LOG . info ( "Canonical URL references a different domain, ignoring in {} " , url ) ; } } catch ( MalformedURLException e ) { LOG . error ( "Malformed canonical URL {} was found in {} " , canonicalValue , url ) ; } return url ; }


Input: Returns a trimmed string or the original one if it is below the threshold set in the configuration . [CODESPLIT] protected String trimText ( String text ) { if ( maxLengthText == - 1 ) return text ; if ( text == null ) return text ; if ( text . length ( ) <= maxLengthText ) return text ; return text . substring ( 0 , maxLengthText ) ; }


Input: / * ( non - Javadoc ) [CODESPLIT] @ SuppressWarnings ( "rawtypes" ) @ Override public void init ( Map stormConf ) { defaultfetchInterval = ConfUtils . getInt ( stormConf , Constants . defaultFetchIntervalParamName , 1440 ) ; fetchErrorFetchInterval = ConfUtils . getInt ( stormConf , Constants . fetchErrorFetchIntervalParamName , 120 ) ; errorFetchInterval = ConfUtils . getInt ( stormConf , Constants . errorFetchIntervalParamName , 44640 ) ; // loads any custom key values // must be of form fetchInterval(.STATUS)?.keyname=value // e.g. fetchInterval.isFeed=true // e.g. fetchInterval.FETCH_ERROR.isFeed=true Map < String , CustomInterval > intervals = new HashMap <> ( ) ; Pattern pattern = Pattern . compile ( "^fetchInterval(\\..+?)?\\.(.+)=(.+)" ) ; Iterator < String > keyIter = stormConf . keySet ( ) . iterator ( ) ; while ( keyIter . hasNext ( ) ) { String key = keyIter . next ( ) ; Matcher m = pattern . matcher ( key ) ; if ( ! m . matches ( ) ) { continue ; } Status status = null ; // was a status specified? if ( m . group ( 1 ) != null ) { status = Status . valueOf ( m . group ( 1 ) . substring ( 1 ) ) ; } String mdname = m . group ( 2 ) ; String mdvalue = m . group ( 3 ) ; int customInterval = ConfUtils . getInt ( stormConf , key , - 1 ) ; if ( customInterval != - 1 ) { CustomInterval interval = intervals . get ( mdname + mdvalue ) ; if ( interval == null ) { interval = new CustomInterval ( mdname , mdvalue , status , customInterval ) ; } else { interval . setDurationForStatus ( status , customInterval ) ; } // specify particular interval for this status intervals . put ( mdname + mdvalue , interval ) ; } } customIntervals = intervals . values ( ) . toArray ( new CustomInterval [ intervals . size ( ) ] ) ; }


Input: / * ( non - Javadoc ) [CODESPLIT] @ Override public Date schedule ( Status status , Metadata metadata ) { int minutesIncrement = 0 ; Optional < Integer > customInterval = Optional . empty ( ) ; // try with a value set in the metadata String customInMetadata = metadata . getFirstValue ( DELAY_METADATA ) ; if ( customInMetadata != null ) { customInterval = Optional . of ( Integer . parseInt ( customInMetadata ) ) ; } // try with the rules from the configuration if ( ! customInterval . isPresent ( ) ) { customInterval = checkCustomInterval ( metadata , status ) ; } if ( customInterval . isPresent ( ) ) { minutesIncrement = customInterval . get ( ) ; } else { switch ( status ) { case FETCHED : minutesIncrement = defaultfetchInterval ; break ; case FETCH_ERROR : minutesIncrement = fetchErrorFetchInterval ; break ; case ERROR : minutesIncrement = errorFetchInterval ; break ; case REDIRECTION : minutesIncrement = defaultfetchInterval ; break ; default : // leave it to now e.g. DISCOVERED } } // a value of -1 means never fetch // we use a conventional value if ( minutesIncrement == - 1 ) { return NEVER ; } Calendar cal = Calendar . getInstance ( ) ; cal . add ( Calendar . MINUTE , minutesIncrement ) ; return cal . getTime ( ) ; }


Input: Returns the first matching custom interval [CODESPLIT] protected final Optional < Integer > checkCustomInterval ( Metadata metadata , Status s ) { if ( customIntervals == null ) return Optional . empty ( ) ; for ( CustomInterval customInterval : customIntervals ) { String [ ] values = metadata . getValues ( customInterval . key ) ; if ( values == null ) { continue ; } for ( String v : values ) { if ( v . equals ( customInterval . value ) ) { return customInterval . getDurationForStatus ( s ) ; } } } return Optional . empty ( ) ; }


Input: Loads and configure the URLFilters based on the storm config if there is one otherwise returns an empty URLFilter . [CODESPLIT] public static URLFilters fromConf ( Map stormConf ) { String configFile = ConfUtils . getString ( stormConf , "urlfilters.config.file" ) ; if ( StringUtils . isNotBlank ( configFile ) ) { try { return new URLFilters ( stormConf , configFile ) ; } catch ( IOException e ) { String message = "Exception caught while loading the URLFilters from " + configFile ; LOG . error ( message ) ; throw new RuntimeException ( message , e ) ; } } return URLFilters . emptyURLFilters ; }


Input: Returns a Scheduler instance based on the configuration * [CODESPLIT] @ SuppressWarnings ( { "rawtypes" , "unchecked" } ) public static Scheduler getInstance ( Map stormConf ) { Scheduler scheduler ; String className = ConfUtils . getString ( stormConf , schedulerClassParamName ) ; if ( StringUtils . isBlank ( className ) ) { throw new RuntimeException ( "Missing value for config  " + schedulerClassParamName ) ; } try { Class < ? > schedulerc = Class . forName ( className ) ; boolean interfaceOK = Scheduler . class . isAssignableFrom ( schedulerc ) ; if ( ! interfaceOK ) { throw new RuntimeException ( "Class " + className + " must extend Scheduler" ) ; } scheduler = ( Scheduler ) schedulerc . newInstance ( ) ; } catch ( Exception e ) { throw new RuntimeException ( "Can't instanciate " + className ) ; } scheduler . init ( stormConf ) ; return scheduler ; }


Input: Submits the topology with the name taken from the configuration * [CODESPLIT] protected int submit ( Config conf , TopologyBuilder builder ) { String name = ConfUtils . getString ( conf , Config . TOPOLOGY_NAME ) ; if ( StringUtils . isBlank ( name ) ) throw new RuntimeException ( "No value found for " + Config . TOPOLOGY_NAME ) ; return submit ( name , conf , builder ) ; }


Input: Submits the topology under a specific name * [CODESPLIT] protected int submit ( String name , Config conf , TopologyBuilder builder ) { // register Metadata for serialization with FieldsSerializer Config . registerSerialization ( conf , Metadata . class ) ; if ( isLocal ) { LocalCluster cluster = new LocalCluster ( ) ; cluster . submitTopology ( name , conf , builder . createTopology ( ) ) ; if ( ttl != - 1 ) { Utils . sleep ( ttl * 1000 ) ; cluster . shutdown ( ) ; } } else { try { StormSubmitter . submitTopology ( name , conf , builder . createTopology ( ) ) ; } catch ( Exception e ) { e . printStackTrace ( ) ; return - 1 ; } } return 0 ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > List < T > fromExcel ( final File file , final Class < T > type ) { final ArrayList < T > list = new ArrayList <> ( ) ; fromExcel ( file , type , list :: add ) ; return list ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > void fromExcel ( final File file , final Class < T > type , final Consumer < ? super T > consumer ) { final Unmarshaller unmarshaller = deserializer ( file , PoijiOptionsBuilder . settings ( ) . build ( ) ) ; unmarshaller . unmarshal ( type , consumer ) ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > List < T > fromExcel ( final InputStream inputStream , PoijiExcelType excelType , final Class < T > type ) { final ArrayList < T > list = new ArrayList <> ( ) ; fromExcel ( inputStream , excelType , type , list :: add ) ; return list ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > void fromExcel ( final InputStream inputStream , PoijiExcelType excelType , final Class < T > type , final Consumer < ? super T > consumer ) { Objects . requireNonNull ( excelType ) ; final Unmarshaller unmarshaller = deserializer ( inputStream , excelType , PoijiOptionsBuilder . settings ( ) . build ( ) ) ; unmarshaller . unmarshal ( type , consumer ) ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > void fromExcel ( final File file , final Class < T > type , final PoijiOptions options , final Consumer < ? super T > consumer ) { final Unmarshaller unmarshaller = deserializer ( file , options ) ; unmarshaller . unmarshal ( type , consumer ) ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > List < T > fromExcel ( final InputStream inputStream , final PoijiExcelType excelType , final Class < T > type , final PoijiOptions options ) { Objects . requireNonNull ( excelType ) ; final ArrayList < T > list = new ArrayList <> ( ) ; fromExcel ( inputStream , excelType , type , options , list :: add ) ; return list ; }


Input: converts excel rows into a list of objects [CODESPLIT] public static synchronized < T > void fromExcel ( final InputStream inputStream , final PoijiExcelType excelType , final Class < T > type , final PoijiOptions options , final Consumer < ? super T > consumer ) { Objects . requireNonNull ( excelType ) ; final Unmarshaller unmarshaller = deserializer ( inputStream , excelType , options ) ; unmarshaller . unmarshal ( type , consumer ) ; }


Input: Using this to hold inner objects that will be mapped to the main object * [CODESPLIT] private Object getInstance ( Field field ) { Object ins = null ; try { if ( fieldInstances . containsKey ( field . getName ( ) ) ) { ins = fieldInstances . get ( field . getName ( ) ) ; } else { ins = field . getType ( ) . getDeclaredConstructor ( ) . newInstance ( ) ; fieldInstances . put ( field . getName ( ) , ins ) ; } } catch ( NoSuchMethodException | InvocationTargetException | IllegalAccessException | InstantiationException e ) { throw new PoijiInstantiationException ( "Cannot create a new instance of " + type . getName ( ) ) ; } return ins ; }


Input: * Modified this method so that for each time it reads a cell it doesn t need to check all fields even after it as found the matching field [CODESPLIT] private boolean setValue ( String content , Class < ? super T > type , int column ) { // For ExcelRow annotation if ( columnToField . containsKey ( - 1 ) ) { Field field = columnToField . get ( - 1 ) ; Object o = casting . castValue ( field . getType ( ) , valueOf ( internalCount ) , options ) ; setFieldData ( field , o , instance ) ; } if ( columnToField . containsKey ( column ) ) { Field field = columnToField . get ( column ) ; if ( columnToSuperClassField . containsKey ( column ) ) { Object ins = null ; ins = getInstance ( columnToSuperClassField . get ( column ) ) ; if ( setValue ( field , column , content , ins ) ) { setFieldData ( columnToSuperClassField . get ( column ) , ins , instance ) ; return true ; } else { return false ; } } return setValue ( field , column , content , instance ) ; } for ( Field field : type . getDeclaredFields ( ) ) { ExcelRow excelRow = field . getAnnotation ( ExcelRow . class ) ; if ( excelRow != null ) { Object o = casting . castValue ( field . getType ( ) , valueOf ( internalCount ) , options ) ; setFieldData ( field , o , instance ) ; columnToField . put ( - 1 , field ) ; } ExcelCellRange range = field . getAnnotation ( ExcelCellRange . class ) ; if ( range != null ) { Object ins = null ; ins = getInstance ( field ) ; for ( Field f : field . getType ( ) . getDeclaredFields ( ) ) { if ( setValue ( f , column , content , ins ) ) { setFieldData ( field , ins , instance ) ; columnToField . put ( column , f ) ; columnToSuperClassField . put ( column , field ) ; return true ; } } } else { if ( setValue ( field , column , content , instance ) ) { columnToField . put ( column , field ) ; return true ; } } } return false ; }


Input: Sets the number of items to be displayed on the wheel . [CODESPLIT] public void setWheelItemCount ( int count ) { mItemCount = count ; mItemAngle = calculateItemAngle ( count ) ; if ( mWheelBounds != null ) { invalidate ( ) ; //TODO ? } }


Input: Taken and modified from Android Source for API < 11 [CODESPLIT] public static int resolveSizeAndState ( int size , int measureSpec ) { int result = size ; int specMode = MeasureSpec . getMode ( measureSpec ) ; int specSize = MeasureSpec . getSize ( measureSpec ) ; switch ( specMode ) { case MeasureSpec . UNSPECIFIED : result = size ; break ; case MeasureSpec . AT_MOST : if ( specSize < size ) { result = specSize ; } else { result = size ; } break ; case MeasureSpec . EXACTLY : result = specSize ; break ; } return result ; }


Input: <p > Sets the wheel s drawable that can also rotate with the items . < / p > <p > Note if the drawable has infinite lines of symmetry then you should set the wheel drawable to not rotate see { @link #setWheelDrawableRotatable ( boolean ) } . In other words if the drawable doesn t look any different whilst it is rotating you should improve the performance by disabling the drawable from rotating . < / p > [CODESPLIT] public void setWheelDrawable ( Drawable drawable ) { mWheelDrawable = drawable ; if ( mWheelBounds != null ) { mWheelDrawable . setBounds ( mWheelBounds . getBoundingRect ( ) ) ; invalidate ( ) ; } }


Input: Sets the empty item drawable that is drawn when outside of the adapter range . [CODESPLIT] public void setEmptyItemDrawable ( Drawable drawable ) { mEmptyItemDrawable = drawable ; EMPTY_CACHE_ITEM . mDrawable = drawable ; if ( mWheelBounds != null ) { invalidate ( ) ; } }


Input: Set the angle of the wheel instantaneously . Note this does not animate to the provided angle . [CODESPLIT] public void setAngle ( float angle ) { mAngle = angle ; updateSelectedPosition ( ) ; if ( mOnAngleChangeListener != null ) { mOnAngleChangeListener . onWheelAngleChange ( mAngle ) ; } invalidate ( ) ; }


Input: Checks to see if the selectedPosition has changed . [CODESPLIT] private void updateSelectedPosition ( ) { int position = ( int ) ( ( - mAngle + - 0.5 * Math . signum ( mAngle ) * mItemAngle ) / mItemAngle ) ; setSelectedPosition ( position ) ; }


Input: Invalidate the drawable at the specific position so that the next Draw call will refresh the Drawable at this given position in the adapter . [CODESPLIT] public void invalidateWheelItemDrawable ( int position ) { int adapterPos = rawPositionToAdapterPosition ( position ) ; if ( isEmptyItemPosition ( adapterPos ) ) return ; CacheItem cacheItem = mItemCacheArray [ adapterPos ] ; if ( cacheItem != null ) cacheItem . mDirty = true ; invalidate ( ) ; }