Input: / * Initializes this JspServlet . [CODESPLIT] public void init ( ServletConfig config ) throws ServletException { super . init ( config ) ; this . config = config ; this . context = config . getServletContext ( ) ; // Initialize the JSP Runtime Context options = new EmbeddedServletOptions ( config , context ) ; rctxt = new JspRuntimeContext ( context , options ) ; // START SJSWS 6232180 // Determine which HTTP methods to service ("*" means all) httpMethodsString = config . getInitParameter ( "httpMethods" ) ; if ( httpMethodsString != null && ! httpMethodsString . equals ( "*" ) ) { httpMethodsSet = new HashSet < String > ( ) ; StringTokenizer tokenizer = new StringTokenizer ( httpMethodsString , ", \t\n\r\f" ) ; while ( tokenizer . hasMoreTokens ( ) ) { httpMethodsSet . add ( tokenizer . nextToken ( ) ) ; } } // END SJSWS 6232180 // START GlassFish 750 taglibs = new ConcurrentHashMap < String , TagLibraryInfo > ( ) ; context . setAttribute ( Constants . JSP_TAGLIBRARY_CACHE , taglibs ) ; tagFileJarUrls = new ConcurrentHashMap < String , URL > ( ) ; context . setAttribute ( Constants . JSP_TAGFILE_JAR_URLS_CACHE , tagFileJarUrls ) ; // END GlassFish 750 if ( log . isLoggable ( Level . FINEST ) ) { log . finest ( Localizer . getMessage ( "jsp.message.scratch.dir.is" , options . getScratchDir ( ) . toString ( ) ) ) ; log . finest ( Localizer . getMessage ( "jsp.message.dont.modify.servlets" ) ) ; } this . jspProbeEmitter = ( JspProbeEmitter ) config . getServletContext ( ) . getAttribute ( "org.glassfish.jsp.monitor.probeEmitter" ) ; }


Input: <p > Look for a <em > precompilation request< / em > as described in Section 8 . 4 . 2 of the JSP 1 . 2 Specification . <strong > WARNING< / strong > - we cannot use <code > request . getParameter () < / code > for this because that will trigger parsing all of the request parameters and not give a servlet the opportunity to call <code > request . setCharacterEncoding () < / code > first . < / p > [CODESPLIT] boolean preCompile ( HttpServletRequest request ) throws ServletException { String queryString = request . getQueryString ( ) ; if ( queryString == null ) { return ( false ) ; } int start = queryString . indexOf ( Constants . PRECOMPILE ) ; if ( start < 0 ) { return ( false ) ; } queryString = queryString . substring ( start + Constants . PRECOMPILE . length ( ) ) ; if ( queryString . length ( ) == 0 ) { return ( true ) ; // ?jsp_precompile } if ( queryString . startsWith ( "&" ) ) { return ( true ) ; // ?jsp_precompile&foo=bar... } if ( ! queryString . startsWith ( "=" ) ) { return ( false ) ; // part of some other name or value } int limit = queryString . length ( ) ; int ampersand = queryString . indexOf ( "&" ) ; if ( ampersand > 0 ) { limit = ampersand ; } String value = queryString . substring ( 1 , limit ) ; if ( value . equals ( "true" ) ) { return ( true ) ; // ?jsp_precompile=true } else if ( value . equals ( "false" ) ) { // Spec says if jsp_precompile=false, the request should not // be delivered to the JSP page; the easiest way to implement // this is to set the flag to true, and precompile the page anyway. // This still conforms to the spec, since it says the // precompilation request can be ignored. return ( true ) ; // ?jsp_precompile=false } else { throw new ServletException ( "Cannot have request parameter " + Constants . PRECOMPILE + " set to " + value ) ; } }


Input: -------------------------------------------------------- Private Methods [CODESPLIT] private void serviceJspFile ( HttpServletRequest request , HttpServletResponse response , String jspUri , Throwable exception , boolean precompile ) throws ServletException , IOException { JspServletWrapper wrapper = ( JspServletWrapper ) rctxt . getWrapper ( jspUri ) ; if ( wrapper == null ) { synchronized ( this ) { wrapper = ( JspServletWrapper ) rctxt . getWrapper ( jspUri ) ; if ( wrapper == null ) { // Check if the requested JSP page exists, to avoid // creating unnecessary directories and files. if ( null == context . getResource ( jspUri ) && ! options . getUsePrecompiled ( ) ) { String includeRequestUri = ( String ) request . getAttribute ( "javax.servlet.include.request_uri" ) ; if ( includeRequestUri != null ) { // Missing JSP resource has been the target of a // RequestDispatcher.include(). // Throw an exception (rather than returning a  // 404 response error code), because any call to // response.sendError() must be ignored by the // servlet engine when issued from within an // included resource (as per the Servlet spec). throw new FileNotFoundException ( JspUtil . escapeXml ( jspUri ) ) ; } response . sendError ( HttpServletResponse . SC_NOT_FOUND ) ; log . severe ( Localizer . getMessage ( "jsp.error.file.not.found" , context . getRealPath ( jspUri ) ) ) ; return ; } boolean isErrorPage = exception != null ; wrapper = new JspServletWrapper ( config , options , jspUri , isErrorPage , rctxt ) ; rctxt . addWrapper ( jspUri , wrapper ) ; } } } wrapper . service ( request , response , precompile ) ; }


Input: STARTS S1AS [CODESPLIT] private void incrementErrorCount ( String jspUri ) { countErrors . incrementAndGet ( ) ; // Fire the jspErrorEvent probe event if ( jspProbeEmitter != null ) { jspProbeEmitter . jspErrorEvent ( jspUri ) ; } }


Input: Clean up timed out user registrations [CODESPLIT] @ Override public void run ( ) throws Exception { this . storageManager . modifyRun ( MODEL_KEY , UserWriteModel . class , users -> { final Date timeout = new Date ( System . currentTimeMillis ( ) - getTimeout ( ) ) ; final Collection < UserEntity > updates = new LinkedList <> ( ) ; final Collection < String > removals = new LinkedList <> ( ) ; for ( final UserEntity user : users . asCollection ( ) ) { if ( user . getEmailTokenDate ( ) == null || user . getEmailTokenDate ( ) . after ( timeout ) ) { continue ; } // process timeout if ( user . isEmailVerified ( ) ) { user . setEmailToken ( null ) ; user . setEmailTokenDate ( null ) ; user . setEmailTokenSalt ( null ) ; updates . add ( user ) ; } else { // delete removals . add ( user . getId ( ) ) ; } } updates . forEach ( users :: putUser ) ; removals . forEach ( users :: removeUser ) ; } ) ; }


Input: Check if the { @code p2 . repo : fragment - keys } field contains a matching key [CODESPLIT] private static boolean hasKey ( final ArtifactInformation art , final String key ) { if ( key == null ) { return false ; } final String keysString = art . getMetaData ( ) . get ( P2RepoConstants . KEY_FRAGMENT_KEYS ) ; if ( keysString == null ) { return false ; } final String [ ] keys = keysString . split ( P2RepoConstants . ENTRY_DELIMITER ) ; for ( final String actualKey : keys ) { if ( key . equals ( actualKey ) ) { return true ; } } return false ; }


Input: Get the last segment of a path the filename [CODESPLIT] public static String getBasename ( final String name ) { if ( name == null ) { return null ; } final String [ ] toks = name . split ( "/" ) ; if ( toks . length < 1 ) { return name ; } return toks [ toks . length - 1 ] ; }


Input: Decompresses into an array of bytes . <p > If <code > len< / code > is zero no bytes are read and <code > 0< / code > is returned . Otherwise this will try to decompress <code > len< / code > bytes of uncompressed data . Less than <code > len< / code > bytes may be read only in the following situations : <ul > <li > The end of the compressed data was reached successfully . < / li > <li > An error is detected after at least one but less <code > len< / code > bytes have already been successfully decompressed . The next call with non - zero <code > len< / code > will immediately throw the pending exception . < / li > <li > An exception is thrown . < / li > < / ul > [CODESPLIT] public int read ( byte [ ] buf , int off , int len ) throws IOException { if ( off < 0 || len < 0 || off + len < 0 || off + len > buf . length ) throw new IndexOutOfBoundsException ( ) ; if ( len == 0 ) return 0 ; if ( in == null ) throw new XZIOException ( "Stream closed" ) ; if ( exception != null ) throw exception ; if ( endReached ) return - 1 ; int size = 0 ; try { while ( len > 0 ) { if ( xzIn == null ) { prepareNextStream ( ) ; if ( endReached ) return size == 0 ? - 1 : size ; } int ret = xzIn . read ( buf , off , len ) ; if ( ret > 0 ) { size += ret ; off += ret ; len -= ret ; } else if ( ret == - 1 ) { xzIn = null ; } } } catch ( IOException e ) { exception = e ; if ( size == 0 ) throw e ; } return size ; }


Input: Returns the number of uncompressed bytes that can be read without blocking . The value is returned with an assumption that the compressed input data will be valid . If the compressed data is corrupt <code > CorruptedInputException< / code > may get thrown before the number of bytes claimed to be available have been read from this input stream . [CODESPLIT] public int available ( ) throws IOException { if ( in == null ) throw new XZIOException ( "Stream closed" ) ; if ( exception != null ) throw exception ; return xzIn == null ? 0 : xzIn . available ( ) ; }


Input: Closes the stream and optionally calls <code > in . close () < / code > . If the stream was already closed this does nothing . If <code > close ( false ) < / code > has been called a further call of <code > close ( true ) < / code > does nothing ( it doesn t call <code > in . close () < / code > ) . <p > If you don t want to close the underlying <code > InputStream< / code > there is usually no need to worry about closing this stream either ; it s fine to do nothing and let the garbage collector handle it . However if you are using { @link ArrayCache } <code > close ( false ) < / code > can be useful to put the allocated arrays back to the cache without closing the underlying <code > InputStream< / code > . <p > Note that if you successfully reach the end of the stream ( <code > read< / code > returns <code > - 1< / code > ) the arrays are automatically put back to the cache by that <code > read< / code > call . In this situation <code > close ( false ) < / code > is redundant ( but harmless ) . [CODESPLIT] public void close ( boolean closeInput ) throws IOException { if ( in != null ) { if ( xzIn != null ) { xzIn . close ( false ) ; xzIn = null ; } try { if ( closeInput ) in . close ( ) ; } finally { in = null ; } } }


Input: Provides a way for the sub class to get a { @link java . nio . ByteBuffer } representation of a certain Rollup object . [CODESPLIT] @ Override protected ByteBuffer toByteBuffer ( Object value ) { if ( ! ( value instanceof BluefloodGaugeRollup ) ) { throw new IllegalArgumentException ( "toByteBuffer(): expecting BluefloodGaugeRollup class but got " + value . getClass ( ) . getSimpleName ( ) ) ; } BluefloodGaugeRollup gaugeRollup = ( BluefloodGaugeRollup ) value ; return serDes . serialize ( gaugeRollup ) ; }


Input: / * Consider 5 files getting merged in the buildstore . Assuming the flush rate to be 1 file / min we need all the files together in order to construct 5 min rollup How are we going to decide that a particular range has been totally filled up and ready to be rolled? One behaviour which we will start seeing in the buildstore is higher ranges starting to build up . This means the current range has almost filled up . But there is still a possibility for backed up data getting merged . So we provide RANGE_BUFFER . Basically for every call to getEligibleData it is going to return ( n - RANGE_BUFFER ) ranges to get rolled up and keep ( RANGE_BUFFER ) in buildstore Also note that returning the range will eventually remove them from buildstore after all rollups are completed in RollupGenerator for that range . [CODESPLIT] public static Map < Range , ConcurrentHashMap < Locator , Points > > getEligibleData ( ) { if ( locatorToTimestampToPoint . size ( ) <= RANGE_BUFFER ) { log . debug ( "Range buffer still not exceeded. Returning null data to rollup generator" ) ; return null ; } else { Object [ ] sortedKeySet = locatorToTimestampToPoint . keySet ( ) . toArray ( ) ; Range cuttingPoint = ( Range ) sortedKeySet [ sortedKeySet . length - RANGE_BUFFER - 1 ] ; log . info ( "Found completed ranges up to the threshold range of {}" , cuttingPoint ) ; completedRangesReturned . mark ( ) ; return locatorToTimestampToPoint . headMap ( cuttingPoint , true ) ; } }


Input: Similar to { @link #get ( Locator String ) } except this does not throw any { @link CacheException } . Instead it may return null if the cache does not contain the requested locator for the specified key . [CODESPLIT] public String safeGet ( Locator locator , String key ) { String cacheValue = null ; try { cacheValue = get ( locator , key ) ; } catch ( CacheException ex ) { log . trace ( String . format ( "no cache value found for locator=%s, key=%s" , locator . toString ( ) , key ) ) ; } return cacheValue ; }


Input: returns true if updated . [CODESPLIT] public boolean put ( Locator locator , String key , String value ) throws CacheException { if ( value == null ) return false ; Timer . Context cachePutTimerContext = MetadataCache . cachePutTimer . time ( ) ; boolean dbWrite = false ; try { CacheKey cacheKey = new CacheKey ( locator , key ) ; String oldValue = cache . getIfPresent ( cacheKey ) ; // don't care if oldValue == EMPTY. // always put new value in the cache. it keeps reads from happening. cache . put ( cacheKey , value ) ; if ( oldValue == null || ! oldValue . equals ( value ) ) { dbWrite = true ; } if ( dbWrite ) { updatedMetricMeter . mark ( ) ; if ( ! batchedWrites ) { databasePut ( locator , key , value ) ; } else { databaseLazyWrite ( locator , key ) ; } } return dbWrite ; } finally { cachePutTimerContext . stop ( ) ; } }


Input: implements the CacheLoader interface . [CODESPLIT] private String databaseLoad ( Locator locator , String key ) throws CacheException { try { CacheKey cacheKey = new CacheKey ( locator , key ) ; Map < String , String > metadata = io . getAllValues ( locator ) ; if ( metadata == null || metadata . isEmpty ( ) ) { cache . put ( cacheKey , NULL ) ; return NULL ; } int metadataRowSize = 0 ; // prepopulate all other metadata other than the key we called the method with for ( Map . Entry < String , String > meta : metadata . entrySet ( ) ) { metadataRowSize += meta . getKey ( ) . getBytes ( ) . length + locator . toString ( ) . getBytes ( ) . length ; if ( meta . getValue ( ) != null ) metadataRowSize += meta . getValue ( ) . getBytes ( ) . length ; if ( meta . getKey ( ) . equals ( key ) ) continue ; CacheKey metaKey = new CacheKey ( locator , meta . getKey ( ) ) ; cache . put ( metaKey , meta . getValue ( ) ) ; } totalMetadataSize . update ( metadataRowSize ) ; String value = metadata . get ( key ) ; if ( value == null ) { cache . put ( cacheKey , NULL ) ; value = NULL ; } return value ; } catch ( IOException ex ) { throw new CacheException ( ex ) ; } }


Input: type comparisions we use to determine how to serialize a number . [CODESPLIT] public static Collection < IMetric > buildMetricsCollection ( AggregatedPayload payload ) { Collection < IMetric > metrics = new ArrayList < IMetric > ( ) ; metrics . addAll ( PreaggregateConversions . convertCounters ( payload . getTenantId ( ) , payload . getTimestamp ( ) , payload . getFlushIntervalMillis ( ) , payload . getCounters ( ) ) ) ; metrics . addAll ( PreaggregateConversions . convertGauges ( payload . getTenantId ( ) , payload . getTimestamp ( ) , payload . getGauges ( ) ) ) ; metrics . addAll ( PreaggregateConversions . convertSets ( payload . getTenantId ( ) , payload . getTimestamp ( ) , payload . getSets ( ) ) ) ; metrics . addAll ( PreaggregateConversions . convertTimers ( payload . getTenantId ( ) , payload . getTimestamp ( ) , payload . getTimers ( ) ) ) ; return metrics ; }


Input: resolve a number to a Long or double . [CODESPLIT] public static Number resolveNumber ( Number n ) { if ( n instanceof LazilyParsedNumber ) { try { return n . longValue ( ) ; } catch ( NumberFormatException ex ) { return n . doubleValue ( ) ; } } else { // already resolved. return n ; } }


Input: History : this used to support multiple types . [CODESPLIT] public ByteBuffer serialize ( String str ) { try { byte [ ] buf = new byte [ computeBufLength ( str ) ] ; CodedOutputStream out = CodedOutputStream . newInstance ( buf ) ; writeToOutputStream ( str , out ) ; return ByteBuffer . wrap ( buf ) ; } catch ( IOException e ) { throw new RuntimeException ( "Error serializing string metadata" , e ) ; } }


Input: writes object to CodedOutputStream . [CODESPLIT] private static void writeToOutputStream ( Object obj , CodedOutputStream out ) throws IOException { out . writeRawByte ( STRING ) ; out . writeStringNoTag ( ( String ) obj ) ; }


Input: Insert a locator with key = shard long value calculated using Util . getShard () [CODESPLIT] @ Override public void insertLocator ( Locator locator ) throws IOException { Timer . Context timer = Instrumentation . getWriteTimerContext ( CassandraModel . CF_METRICS_LOCATOR_NAME ) ; try { MutationBatch mutationBatch = AstyanaxIO . getKeyspace ( ) . prepareMutationBatch ( ) ; AstyanaxWriter . getInstance ( ) . insertLocator ( locator , mutationBatch ) ; mutationBatch . execute ( ) ; } catch ( Exception e ) { throw new IOException ( e ) ; } finally { timer . stop ( ) ; } }


Input: Returns the locators for a shard i . e . those that should be rolled up for a given shard . Should means : 1 ) A locator is capable of rollup . 2 ) A locator has had new data in the past LOCATOR_TTL seconds . [CODESPLIT] @ Override public Collection < Locator > getLocators ( long shard ) throws IOException { Timer . Context ctx = Instrumentation . getReadTimerContext ( CassandraModel . CF_METRICS_LOCATOR_NAME ) ; try { RowQuery < Long , Locator > query = AstyanaxIO . getKeyspace ( ) . prepareQuery ( CassandraModel . CF_METRICS_LOCATOR ) . getKey ( shard ) ; if ( LOG . isTraceEnabled ( ) ) LOG . trace ( "ALocatorIO.getLocators() executing: select * from \"" + CassandraModel . KEYSPACE + "\"." + CassandraModel . CF_METRICS_LOCATOR_NAME + " where key=" + Long . toString ( shard ) ) ; return query . execute ( ) . getResult ( ) . getColumnNames ( ) ; } catch ( NotFoundException e ) { Instrumentation . markNotFound ( CassandraModel . CF_METRICS_LOCATOR_NAME ) ; return Collections . emptySet ( ) ; } catch ( ConnectionException ex ) { Instrumentation . markReadError ( ex ) ; LOG . error ( "Connection exception during getLocators(" + Long . toString ( shard ) + ")" , ex ) ; throw new IOException ( "Error reading locators" , ex ) ; } finally { ctx . stop ( ) ; } }


Input: Gets the TTL for a particular locator rollupType and granularity . [CODESPLIT] protected int getTtl ( Locator locator , RollupType rollupType , Granularity granularity ) { return ( int ) TTL_PROVIDER . getTTL ( locator . getTenantId ( ) , granularity , rollupType ) . get ( ) . toSeconds ( ) ; }


Input: Converts the map of timestamp - > { @link Rollup } to { @link Points } object [CODESPLIT] protected < T extends Object > Points < T > convertToPoints ( final Map < Long , T > timestampToRollupMap ) { Points points = new Points ( ) ; for ( Map . Entry < Long , T > value : timestampToRollupMap . entrySet ( ) ) { points . add ( createPoint ( value . getKey ( ) , value . getValue ( ) ) ) ; } return points ; }


Input: Create all prepared statements use in this class for metrics_locator [CODESPLIT] private void createPreparedStatements ( ) { // create a generic select statement for retrieving from metrics_locator Select . Where select = QueryBuilder . select ( ) . all ( ) . from ( CassandraModel . CF_METRICS_LOCATOR_NAME ) . where ( eq ( KEY , bindMarker ( ) ) ) ; getValue = DatastaxIO . getSession ( ) . prepare ( select ) ; // create a generic insert statement for inserting into metrics_locator Insert insert = QueryBuilder . insertInto ( CassandraModel . CF_METRICS_LOCATOR_NAME ) . using ( ttl ( TenantTtlProvider . LOCATOR_TTL ) ) . value ( KEY , bindMarker ( ) ) . value ( COLUMN1 , bindMarker ( ) ) . value ( VALUE , bindMarker ( ) ) ; putValue = DatastaxIO . getSession ( ) . prepare ( insert ) . setConsistencyLevel ( ConsistencyLevel . LOCAL_ONE ) ; }


Input: Insert a locator with key = shard long value calculated using Util . getShard () [CODESPLIT] @ Override public void insertLocator ( Locator locator ) throws IOException { Session session = DatastaxIO . getSession ( ) ; Timer . Context timer = Instrumentation . getWriteTimerContext ( CassandraModel . CF_METRICS_LOCATOR_NAME ) ; try { // bound values and execute BoundStatement bs = getBoundStatementForLocator ( locator ) ; session . execute ( bs ) ; } finally { timer . stop ( ) ; } }


Input: package protect so other classes in this package can call it [CODESPLIT] BoundStatement getBoundStatementForLocator ( Locator locator ) { // get shard this locator would belong to long shard = ( long ) Util . getShard ( locator . toString ( ) ) ; return putValue . bind ( shard , locator . toString ( ) , "" ) ; }


Input: Returns the locators for a shard i . e . those that should be rolled up for a given shard . Should means : 1 ) A locator is capable of rollup . 2 ) A locator has had new data in the past LOCATOR_TTL seconds . [CODESPLIT] @ Override public Collection < Locator > getLocators ( long shard ) throws IOException { Timer . Context ctx = Instrumentation . getReadTimerContext ( CassandraModel . CF_METRICS_LOCATOR_NAME ) ; Session session = DatastaxIO . getSession ( ) ; Collection < Locator > locators = new ArrayList < Locator > ( ) ; try { // bind value BoundStatement bs = getValue . bind ( shard ) ; List < Row > results = session . execute ( bs ) . all ( ) ; for ( Row row : results ) { if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "Read metrics_locators with shard " + shard + ": " + row . getString ( KEY ) + row . getString ( COLUMN1 ) ) ; } locators . add ( Locator . createLocatorFromDbKey ( row . getString ( COLUMN1 ) ) ) ; } // return results if ( locators . size ( ) == 0 ) { Instrumentation . markNotFound ( CassandraModel . CF_METRICS_LOCATOR_NAME ) ; return Collections . emptySet ( ) ; } else { return locators ; } } finally { ctx . stop ( ) ; } }


Input: This method is used to log delayed metrics if tracking delayed metrics is turned on for this Blueflood service . [CODESPLIT] public void trackDelayedMetricsTenant ( String tenantid , final List < Metric > delayedMetrics ) { if ( isTrackingDelayedMetrics ) { String logMessage = String . format ( "[TRACKER][DELAYED METRIC] Tenant sending delayed metrics %s" , tenantid ) ; log . info ( logMessage ) ; // log individual delayed metrics locator and collectionTime double delayedMinutes ; long nowMillis = System . currentTimeMillis ( ) ; for ( Metric metric : delayedMetrics ) { delayedMinutes = ( double ) ( nowMillis - metric . getCollectionTime ( ) ) / 1000 / 60 ; logMessage = String . format ( "[TRACKER][DELAYED METRIC] %s has collectionTime %s which is delayed by %.2f minutes" , metric . getLocator ( ) . toString ( ) , dateFormatter . format ( new Date ( metric . getCollectionTime ( ) ) ) , delayedMinutes ) ; log . info ( logMessage ) ; } } }


Input: This method logs the delayed aggregated metrics for a particular tenant if tracking delayed metric is turned on for this Blueflood service . Aggregated metrics have one single timestamp for the group of metrics that are sent in one request . [CODESPLIT] public void trackDelayedAggregatedMetricsTenant ( String tenantId , long collectionTimeMs , long delayTimeMs , List < String > delayedMetricNames ) { if ( isTrackingDelayedMetrics ) { String logMessage = String . format ( "[TRACKER][DELAYED METRIC] Tenant sending delayed metrics %s" , tenantId ) ; log . info ( logMessage ) ; // log individual delayed metrics locator and collectionTime double delayMin = delayTimeMs / 1000 / 60 ; logMessage = String . format ( "[TRACKER][DELAYED METRIC] %s have collectionTime %s which is delayed by %.2f minutes" , StringUtils . join ( delayedMetricNames , "," ) , dateFormatter . format ( new Date ( collectionTimeMs ) ) , delayMin ) ; log . info ( logMessage ) ; } }


Input: read out a type - specified number . [CODESPLIT] protected Number getUnversionedDoubleOrLong ( CodedInputStream in ) throws IOException { byte type = in . readRawByte ( ) ; if ( type == Constants . B_DOUBLE ) return in . readDouble ( ) ; else return in . readRawVarint64 ( ) ; }


Input: put out a number prefaced only by a type . [CODESPLIT] protected void putUnversionedDoubleOrLong ( Number number , CodedOutputStream out ) throws IOException { if ( number instanceof Double ) { out . writeRawByte ( Constants . B_DOUBLE ) ; out . writeDoubleNoTag ( number . doubleValue ( ) ) ; } else { out . writeRawByte ( Constants . B_I64 ) ; out . writeRawVarint64 ( number . longValue ( ) ) ; } }


Input: Convert the existing configuration values into a Map including those specified in defaultProps . [CODESPLIT] public Map < Object , Object > getAllProperties ( ) { Map < Object , Object > map = new HashMap < Object , Object > ( ) ; for ( Object key : defaultProps . keySet ( ) ) { map . put ( key , defaultProps . getProperty ( key . toString ( ) ) ) ; } for ( Object key : props . keySet ( ) ) { map . put ( key , props . getProperty ( key . toString ( ) ) ) ; } return Collections . unmodifiableMap ( map ) ; }


Input: idempotent other than when the month changes between two calls [CODESPLIT] private void createContainer ( ) { String containerName = CONTAINER_DATE_FORMAT . format ( new Date ( ) ) ; blobStore . createContainerInLocation ( null , containerName ) ; lastContainerCreated = containerName ; }


Input: Provides a way for the sub class to get a { @link java . nio . ByteBuffer } representation of a certain Rollup object . [CODESPLIT] @ Override protected ByteBuffer toByteBuffer ( Object value ) { if ( ! ( value instanceof BluefloodCounterRollup ) ) { throw new IllegalArgumentException ( "toByteBuffer(): expecting BluefloodCounterRollup class but got " + value . getClass ( ) . getSimpleName ( ) ) ; } BluefloodCounterRollup counterRollup = ( BluefloodCounterRollup ) value ; return serDes . serialize ( counterRollup ) ; }


Input: Retrieves the { [CODESPLIT] @ Override protected BoundStatement getBoundStatementForMetric ( IMetric metric , Granularity granularity ) { Object metricValue = metric . getMetricValue ( ) ; if ( ! ( metricValue instanceof BluefloodCounterRollup ) ) { throw new InvalidDataException ( String . format ( "getBoundStatementForMetric(locator=%s, granularity=%s): metric value %s is not type BluefloodCounterRollup" , metric . getLocator ( ) , granularity , metric . getMetricValue ( ) . getClass ( ) . getSimpleName ( ) ) ) ; } PreparedStatement statement = metricsCFPreparedStatements . preaggrGranToInsertStatement . get ( granularity ) ; return statement . bind ( metric . getLocator ( ) . toString ( ) , metric . getCollectionTime ( ) , serDes . serialize ( ( BluefloodCounterRollup ) metricValue ) , metric . getTtlInSeconds ( ) ) ; }


Input: { [CODESPLIT] public void update ( long millis , int shard ) { // there are two update paths. for managed shards, we must guard the // scheduled and running collections. but for unmanaged shards, we just // let the update happen uncontested. final Timer . Context dirtyTimerCtx = markSlotDirtyTimer . time ( ) ; try { if ( log . isTraceEnabled ( ) ) { log . trace ( "Updating {} to {}" , shard , millis ) ; } boolean isManaged = shardStateManager . contains ( shard ) ; for ( Granularity g : Granularity . rollupGranularities ( ) ) { ShardStateManager . SlotStateManager slotStateManager = shardStateManager . getSlotStateManager ( shard , g ) ; int slot = g . slot ( millis ) ; if ( isManaged ) { synchronized ( scheduledSlots ) { //put SlotKey key = SlotKey . of ( g , slot , shard ) ; if ( scheduledSlots . remove ( key ) && log . isDebugEnabled ( ) ) { // don't worry about orderedScheduledSlots log . debug ( "descheduled {}." , key ) ; } } } slotStateManager . createOrUpdateForSlotAndMillisecond ( slot , millis ) ; } } finally { dirtyTimerCtx . stop ( ) ; } }


Input: only one thread should be calling in this puppy . [CODESPLIT] void scheduleEligibleSlots ( long maxAgeMillis , long rollupDelayForMetricsWithShortDelay , long rollupWaitForMetricsWithLongDelay ) { long now = scheduleTime ; ArrayList < Integer > shardKeys = new ArrayList < Integer > ( shardStateManager . getManagedShards ( ) ) ; Collections . shuffle ( shardKeys ) ; for ( int shard : shardKeys ) { for ( Granularity g : Granularity . rollupGranularities ( ) ) { // sync on map since we do not want anything added to or taken from it while we iterate. synchronized ( scheduledSlots ) { // read synchronized ( runningSlots ) { // read List < Integer > slotsToWorkOn = shardStateManager . getSlotStateManager ( shard , g ) . getSlotsEligibleForRollup ( now , maxAgeMillis , rollupDelayForMetricsWithShortDelay , rollupWaitForMetricsWithLongDelay ) ; if ( slotsToWorkOn . size ( ) == 0 ) { continue ; } if ( ! canWorkOnShard ( shard ) ) { continue ; } for ( Integer slot : slotsToWorkOn ) { SlotKey slotKey = SlotKey . of ( g , slot , shard ) ; if ( areChildKeysOrSelfKeyScheduledOrRunning ( slotKey ) ) { continue ; } SlotKey key = SlotKey . of ( g , slot , shard ) ; scheduledSlots . add ( key ) ; orderedScheduledSlots . add ( key ) ; recentlyScheduledShards . put ( shard , scheduleTime ) ; } } } } } }


Input: Returns the next scheduled key . It has a few side effects : 1 ) it resets update tracking for that slot 2 ) it adds the key to the set of running rollups . [CODESPLIT] @ VisibleForTesting SlotKey getNextScheduled ( ) { synchronized ( scheduledSlots ) { if ( scheduledSlots . size ( ) == 0 ) return null ; synchronized ( runningSlots ) { SlotKey key = orderedScheduledSlots . remove ( 0 ) ; int slot = key . getSlot ( ) ; Granularity gran = key . getGranularity ( ) ; int shard = key . getShard ( ) ; // notice how we change the state, but the timestamp remained // the same. this is important.  When the state is evaluated // (i.e., in Reader.getShardState()) we need to realize that // when timestamps are the same (this will happen), that a // remove always wins during the coalesce. scheduledSlots . remove ( key ) ; if ( canWorkOnShard ( shard ) ) { UpdateStamp stamp = shardStateManager . getSlotStateManager ( shard , gran ) . getAndSetState ( slot , UpdateStamp . State . Running ) ; runningSlots . put ( key , stamp . getTimestamp ( ) ) ; return key ; } else { shardOwnershipChanged . mark ( ) ; return null ; } } } }


Input: Take the given slot out of the running group and put it back into the scheduled group . If { @code rescheduleImmediately } is true the slot will be the next slot returned by a call to { @link #getNextScheduled () } . If { @code rescheduleImmediately } is false then the given slot will go to the end of the line as when it was first scheduled by { @link #scheduleEligibleSlots ( long long long ) } . [CODESPLIT] void pushBackToScheduled ( SlotKey key , boolean rescheduleImmediately ) { synchronized ( scheduledSlots ) { synchronized ( runningSlots ) { int slot = key . getSlot ( ) ; Granularity gran = key . getGranularity ( ) ; int shard = key . getShard ( ) ; // no need to set dirty/clean here. shardStateManager . getSlotStateManager ( shard , gran ) . getAndSetState ( slot , UpdateStamp . State . Active ) ; scheduledSlots . add ( key ) ; log . debug ( "pushBackToScheduled -> added to scheduledSlots: " + key + " size:" + scheduledSlots . size ( ) ) ; if ( rescheduleImmediately ) { orderedScheduledSlots . add ( 0 , key ) ; } else { orderedScheduledSlots . add ( key ) ; } } } }


Input: Remove the given slot from the running group after it has been successfully re - rolled . [CODESPLIT] void clearFromRunning ( SlotKey slotKey ) { synchronized ( runningSlots ) { runningSlots . remove ( slotKey ) ; UpdateStamp stamp = shardStateManager . getUpdateStamp ( slotKey ) ; shardStateManager . setAllCoarserSlotsDirtyForSlot ( slotKey ) ; //When state gets set to "X", before it got persisted, it might get scheduled for rollup //again, if we get delayed metrics. To prevent this we temporarily set last rollup time with current //time. This value wont get persisted. long currentTimeInMillis = clock . now ( ) . getMillis ( ) ; stamp . setLastRollupTimestamp ( currentTimeInMillis ) ; log . debug ( "SlotKey {} is marked in memory with last rollup time as {}" , slotKey , currentTimeInMillis ) ; // Update the stamp to Rolled state if and only if the current state // is running. If the current state is active, it means we received // a delayed put which toggled the status to Active. if ( stamp . getState ( ) == UpdateStamp . State . Running ) { stamp . setState ( UpdateStamp . State . Rolled ) ; // Note: Rollup state will be updated to the last ACTIVE // timestamp which caused rollup process to kick in. stamp . setDirty ( true ) ; } } }


Input: This method inserts a collection of { @link com . rackspacecloud . blueflood . types . IMetric } objects to the appropriate Cassandra column family . Effectively this method is only called to write the raw metrics received during Ingest requests . Another method insertRollups is used by the Rollup processes to write rolled up metrics . [CODESPLIT] @ Override public void insertMetrics ( Collection < IMetric > metrics ) throws IOException { Timer . Context ctx = Instrumentation . getWriteTimerContext ( CassandraModel . CF_METRICS_FULL_NAME ) ; try { if ( isBatchIngestEnabled ) { insertMetricsInBatch ( metrics ) ; } else { insertMetricsIndividually ( metrics ) ; } } finally { ctx . stop ( ) ; } }


Input: Fetches { @link com . rackspacecloud . blueflood . outputs . formats . MetricData } objects for the specified list of { @link com . rackspacecloud . blueflood . types . Locator } and { @link com . rackspacecloud . blueflood . types . Range } from the specified column family [CODESPLIT] @ Override public Map < Locator , MetricData > getDatapointsForRange ( List < Locator > locators , Range range , Granularity gran ) { Map < Locator , MetricData > metrics = new HashMap < Locator , MetricData > ( ) ; String columnFamily = CassandraModel . getBasicColumnFamilyName ( gran ) ; metrics . putAll ( super . getDatapointsForRange ( locators , range , columnFamily , gran ) ) ; return metrics ; }


Input: Return the appropriate IO object which interacts with the Cassandra database . [CODESPLIT] @ Override public DAbstractMetricIO getIO ( String rollupType , Granularity granularity ) { if ( granularity == Granularity . FULL ) return simpleNumberIO ; else return basicIO ; }


Input: Inserts a collection of metrics in a batch using an unlogged { @link BatchStatement } [CODESPLIT] private void insertMetricsInBatch ( Collection < IMetric > metrics ) throws IOException { BatchStatement batch = new BatchStatement ( BatchStatement . Type . UNLOGGED ) ; for ( IMetric metric : metrics ) { BoundStatement bound = simpleNumberIO . getBoundStatementForMetric ( metric ) ; batch . add ( bound ) ; Instrumentation . markFullResMetricWritten ( ) ; Locator locator = metric . getLocator ( ) ; if ( ! LocatorCache . getInstance ( ) . isLocatorCurrentInBatchLayer ( locator ) ) { LocatorCache . getInstance ( ) . setLocatorCurrentInBatchLayer ( locator ) ; batch . add ( locatorIO . getBoundStatementForLocator ( locator ) ) ; } // if we are recording delayed metrics, we may need to do an // extra insert if ( isRecordingDelayedMetrics ) { BoundStatement bs = getBoundStatementForMetricIfDelayed ( metric ) ; if ( bs != null ) { batch . add ( bs ) ; } } } LOG . trace ( String . format ( "insert batch statement size=%d" , batch . size ( ) ) ) ; try { DatastaxIO . getSession ( ) . execute ( batch ) ; } catch ( Exception ex ) { Instrumentation . markWriteError ( ) ; LOG . error ( String . format ( "error writing batch of %d metrics" , batch . size ( ) ) , ex ) ; } }


Input: Listens on the event . [CODESPLIT] public Emitter on ( String event , Listener fn ) { ConcurrentLinkedQueue < Listener > callbacks = this . callbacks . get ( event ) ; if ( callbacks == null ) { callbacks = new ConcurrentLinkedQueue < Listener > ( ) ; ConcurrentLinkedQueue < Listener > _callbacks = this . callbacks . putIfAbsent ( event , callbacks ) ; if ( _callbacks != null ) { callbacks = _callbacks ; } } callbacks . add ( fn ) ; return this ; }


Input: Adds a one time listener for the event . [CODESPLIT] public Emitter once ( final String event , final Listener < T > fn ) { Listener on = new Listener < T > ( ) { @ Override public void call ( T ... args ) { Emitter . this . off ( event , this ) ; fn . call ( args ) ; } } ; this . onceCallbacks . put ( fn , on ) ; this . on ( event , on ) ; return this ; }


Input: Removes all listeners of the specified event . [CODESPLIT] public Emitter off ( String event ) { ConcurrentLinkedQueue < Listener > callbacks = this . callbacks . remove ( event ) ; if ( callbacks != null ) { for ( Listener fn : callbacks ) { this . onceCallbacks . remove ( fn ) ; } } return this ; }


Input: Executes each of listeners with the given args . [CODESPLIT] public Future emit ( String event , T ... args ) { ConcurrentLinkedQueue < Listener > callbacks = this . callbacks . get ( event ) ; if ( callbacks != null ) { callbacks = new ConcurrentLinkedQueue < Listener > ( callbacks ) ; for ( Listener fn : callbacks ) { fn . call ( args ) ; } } return null ; }


Input: Returns a list of listeners for the specified event . [CODESPLIT] public List < Listener > listeners ( String event ) { ConcurrentLinkedQueue < Listener > callbacks = this . callbacks . get ( event ) ; return callbacks != null ? new ArrayList < Listener > ( callbacks ) : new ArrayList < Listener > ( ) ; }


Input: Get the remote file name . [CODESPLIT] public String getRemoteName ( ) { Date time = new Date ( timestamp ) ; String formattedTime = new SimpleDateFormat ( "yyyyMMdd_" ) . format ( time ) ; return formattedTime + System . currentTimeMillis ( ) + "_" + Configuration . getInstance ( ) . getStringProperty ( CloudfilesConfig . CLOUDFILES_HOST_UNIQUE_IDENTIFIER ) ; }


Input: Serialize a Rollup Event and append it to the file . [CODESPLIT] public void append ( RollupEvent rollup ) throws IOException { ensureOpen ( ) ; outputStream . write ( serializer . toBytes ( rollup ) ) ; outputStream . write ( ' ' ) ; outputStream . flush ( ) ; }


Input: Parse the timestamp from a filename . [CODESPLIT] private static long parseTimestamp ( String fileName ) throws NumberFormatException { String numberPart = fileName . substring ( 0 , fileName . length ( ) - 5 ) ; return Long . parseLong ( numberPart ) ; }


Input: Starts the Ingest server [CODESPLIT] public void startServer ( ) throws InterruptedException { RouteMatcher router = new RouteMatcher ( ) ; router . get ( "/v1.0" , new DefaultHandler ( ) ) ; router . post ( "/v1.0/multitenant/experimental/metrics" , new HttpMultitenantMetricsIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . post ( "/v1.0/:tenantId/experimental/metrics" , new HttpMetricsIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . post ( "/v1.0/:tenantId/experimental/metrics/statsd" , new HttpAggregatedIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . get ( "/v2.0" , new DefaultHandler ( ) ) ; router . post ( "/v2.0/:tenantId/ingest/multi" , new HttpMultitenantMetricsIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . post ( "/v2.0/:tenantId/ingest" , new HttpMetricsIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . post ( "/v2.0/:tenantId/ingest/aggregated" , new HttpAggregatedIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . post ( "/v2.0/:tenantId/ingest/aggregated/multi" , new HttpAggregatedMultiIngestionHandler ( processor , timeout , ENABLE_PER_TENANT_METRICS ) ) ; router . post ( "/v2.0/:tenantId/events" , getHttpEventsIngestionHandler ( ) ) ; final RouteMatcher finalRouter = router ; log . info ( "Starting metrics listener HTTP server on port {}" , httpIngestPort ) ; ServerBootstrap server = new ServerBootstrap ( ) ; server . group ( acceptorGroup , workerGroup ) . channel ( NioServerSocketChannel . class ) . childHandler ( new ChannelInitializer < SocketChannel > ( ) { @ Override public void initChannel ( SocketChannel channel ) throws Exception { setupPipeline ( channel , finalRouter ) ; } } ) ; Channel channel = server . bind ( new InetSocketAddress ( httpIngestHost , httpIngestPort ) ) . sync ( ) . channel ( ) ; allOpenChannels . add ( channel ) ; //register the tracker MBean for JMX/jolokia log . info ( "Registering tracker service" ) ; Tracker . getInstance ( ) . register ( ) ; log . info ( "Token search improvements enabled: " + EXP_TOKEN_SEARCH_IMPROVEMENTS ) ; }


Input: determine which DataType to use for serialization . [CODESPLIT] public static Rollup . Type getRollupComputer ( RollupType srcType , Granularity srcGran ) { switch ( srcType ) { case COUNTER : return Rollup . CounterFromCounter ; case TIMER : return Rollup . TimerFromTimer ; case GAUGE : return Rollup . GaugeFromGauge ; case BF_BASIC : return srcGran == Granularity . FULL ? Rollup . BasicFromRaw : Rollup . BasicFromBasic ; case SET : return Rollup . SetFromSet ; default : break ; } throw new IllegalArgumentException ( String . format ( "Cannot compute rollups for %s from %s" , srcType . name ( ) , srcGran . shortName ( ) ) ) ; }


Input: Returns an instance of this class based on what configuration says our driver should be . [CODESPLIT] public static synchronized IOContainer fromConfig ( ) { if ( FROM_CONFIG_INSTANCE == null ) { String driver = configuration . getStringProperty ( CoreConfig . CASSANDRA_DRIVER ) ; LOG . info ( String . format ( "Using driver %s" , driver ) ) ; boolean isRecordingDelayedMetrics = configuration . getBooleanProperty ( CoreConfig . RECORD_DELAYED_METRICS ) ; LOG . info ( String . format ( "Recording delayed metrics: %s" , isRecordingDelayedMetrics ) ) ; boolean isDtxIngestBatchEnabled = configuration . getBooleanProperty ( CoreConfig . ENABLE_DTX_INGEST_BATCH ) ; LOG . info ( String . format ( "Datastax Ingest batch enabled: %s" , isDtxIngestBatchEnabled ) ) ; FROM_CONFIG_INSTANCE = new IOContainer ( DriverType . getDriverType ( driver ) , isRecordingDelayedMetrics , isDtxIngestBatchEnabled ) ; } return FROM_CONFIG_INSTANCE ; }


Input: Helper function to build the ttl mapping . Only insert to the mapping if the value is a valid date . [CODESPLIT] private boolean put ( ImmutableTable . Builder < Granularity , RollupType , TimeValue > ttlMapBuilder , Configuration config , Granularity gran , RollupType rollupType , TtlConfig configKey ) { int value ; try { value = config . getIntegerProperty ( configKey ) ; if ( value < 0 ) return false ; } catch ( NumberFormatException ex ) { log . trace ( String . format ( "No valid TTL config set for granularity: %s, rollup type: %s" , gran . name ( ) , rollupType . name ( ) ) , ex ) ; return false ; } ttlMapBuilder . put ( gran , rollupType , new TimeValue ( value , TimeUnit . DAYS ) ) ; return true ; }


Input: merge simple numbers with this rollup . [CODESPLIT] protected void computeFromSimpleMetrics ( Points < SimpleNumber > input ) throws IOException { if ( input == null ) { throw new IOException ( "Null input to create rollup from" ) ; } if ( input . isEmpty ( ) ) { return ; } Map < Long , Points . Point < SimpleNumber > > points = input . getPoints ( ) ; for ( Map . Entry < Long , Points . Point < SimpleNumber > > item : points . entrySet ( ) ) { this . count += 1 ; SimpleNumber numericMetric = item . getValue ( ) . getData ( ) ; average . handleFullResMetric ( numericMetric . getValue ( ) ) ; variance . handleFullResMetric ( numericMetric . getValue ( ) ) ; minValue . handleFullResMetric ( numericMetric . getValue ( ) ) ; maxValue . handleFullResMetric ( numericMetric . getValue ( ) ) ; } }


Input: merge rollups into this rollup . [CODESPLIT] protected void computeFromRollupsHelper ( Points < ? extends IBaseRollup > input ) throws IOException { if ( input == null ) { throw new IOException ( "Null input to create rollup from" ) ; } if ( input . isEmpty ( ) ) { return ; } // See this and get mind blown: // http://stackoverflow.com/questions/18907262/bounded-wildcard-related-compiler-error Map < Long , ? extends Points . Point < ? extends IBaseRollup > > points = input . getPoints ( ) ; for ( Map . Entry < Long , ? extends Points . Point < ? extends IBaseRollup > > item : points . entrySet ( ) ) { IBaseRollup rollup = item . getValue ( ) . getData ( ) ; if ( ! ( rollup instanceof BaseRollup ) ) { throw new IOException ( "Cannot create BaseRollup from type " + rollup . getClass ( ) . getName ( ) ) ; } IBaseRollup baseRollup = ( IBaseRollup ) rollup ; this . count += baseRollup . getCount ( ) ; average . handleRollupMetric ( baseRollup ) ; variance . handleRollupMetric ( baseRollup ) ; minValue . handleRollupMetric ( baseRollup ) ; maxValue . handleRollupMetric ( baseRollup ) ; } }


Input: For a particular rollupType determine what is the proper class to do read / write with and return that [CODESPLIT] public static AbstractMetricsRW getMetricsRWForRollupType ( RollupType rollupType ) { if ( rollupType == null || rollupType == RollupType . BF_BASIC ) { return IOContainer . fromConfig ( ) . getBasicMetricsRW ( ) ; } else { return IOContainer . fromConfig ( ) . getPreAggregatedMetricsRW ( ) ; } }


Input: package protect so other classes in this package can call it [CODESPLIT] BoundStatement getBoundStatementForLocator ( Granularity granularity , int slot , Locator locator ) { int shard = Util . getShard ( locator . toString ( ) ) ; return putValue . bind ( SlotKey . of ( granularity , slot , shard ) . toString ( ) , locator . toString ( ) , "" ) ; }


Input: compute the maximum width for each field across a collection of formatters . [CODESPLIT] public static int [ ] computeMaximums ( String [ ] headers , OutputFormatter ... outputs ) { int [ ] max = new int [ headers . length ] ; for ( int i = 0 ; i < headers . length ; i ++ ) max [ i ] = headers [ i ] . length ( ) ; for ( OutputFormatter output : outputs ) { max [ 0 ] = Math . max ( output . host . length ( ) , max [ 0 ] ) ; for ( int i = 1 ; i < headers . length ; i ++ ) max [ i ] = Math . max ( output . results [ i - 1 ] . length ( ) , max [ i ] ) ; } return max ; }


Input: formats a header row after maximums have been established . [CODESPLIT] public static String formatHeader ( int [ ] maximums , String [ ] headers ) { StringBuilder sb = new StringBuilder ( ) ; for ( int i = 0 ; i < headers . length ; i ++ ) sb = sb . append ( formatIn ( headers [ i ] , maximums [ i ] , false ) ) . append ( GAP ) ; return sb . toString ( ) ; }


Input: formats results and sets formattedStrings . [CODESPLIT] public static String [ ] format ( int [ ] maximums , OutputFormatter ... outputs ) { String [ ] formattedStrings = new String [ outputs . length ] ; int pos = 0 ; for ( OutputFormatter output : outputs ) { StringBuilder sb = new StringBuilder ( ) ; sb = sb . append ( formatIn ( output . host , maximums [ 0 ] , false ) ) ; for ( int i = 0 ; i < output . results . length ; i ++ ) sb = sb . append ( GAP ) . append ( formatIn ( output . results [ i ] , maximums [ i + 1 ] , true ) ) ; formattedStrings [ pos ++ ] = sb . toString ( ) ; } return formattedStrings ; }


Input: Registers the different ZooKeeper metrics . [CODESPLIT] private void registerMetrics ( final ObjectName nameObj , MetricRegistry reg ) { reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Lock Disinterested Time Millis" ) , new JmxAttributeGauge ( nameObj , "LockDisinterestedTimeMillis" ) ) ; reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Min Lock Hold Time Millis" ) , new JmxAttributeGauge ( nameObj , "MinLockHoldTimeMillis" ) ) ; reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Seconds Since Last Scavenge" ) , new JmxAttributeGauge ( nameObj , "SecondsSinceLastScavenge" ) ) ; reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Zk Connection Status" ) , new JmxAttributeGauge ( nameObj , "ZkConnectionStatus" ) { @ Override public Object getValue ( ) { Object val = super . getValue ( ) ; if ( val . equals ( "connected" ) ) { return 1 ; } return 0 ; } } ) ; reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Held Shards" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return getHeldShards ( ) . size ( ) ; } } ) ; reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Unheld Shards" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return getUnheldShards ( ) . size ( ) ; } } ) ; reg . register ( MetricRegistry . name ( ZKShardLockManager . class , "Error Shards" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return getErrorShards ( ) . size ( ) ; } } ) ; }


Input: Waits until the zookeeper connection is available . [CODESPLIT] @ VisibleForTesting boolean waitForZKConnections ( long waitTimeSeconds ) { for ( int i = 0 ; i < waitTimeSeconds ; i ++ ) { if ( connected ) { return connected ; } log . debug ( "Waiting for connect" ) ; try { Thread . sleep ( 1000 ) ; } catch ( InterruptedException ex ) { } } return connected ; }


Input: Only called from { @link #init ( TimeValue ) } . [CODESPLIT] @ VisibleForTesting void prefetchLocks ( ) { if ( ! connected ) { log . warn ( "Cannot connect to Zookeeper; will not perform initial lock acquisition" ) ; for ( Lock lock : locks . values ( ) ) { lock . connectionLost ( ) ; } } else { log . info ( "Pre-fetching zookeeper locks for shards" ) ; boolean isManagingAllShards = locks . size ( ) >= Constants . NUMBER_OF_SHARDS ; int maxLocksToPrefetch = moreThanHalf ( ) ; if ( isManagingAllShards ) { maxLocksToPrefetch = Constants . NUMBER_OF_SHARDS ; } List < Integer > shards = new ArrayList < Integer > ( locks . keySet ( ) ) ; Collections . shuffle ( shards ) ; int locksObtained = 0 ; for ( int shard : shards ) { try { log . debug ( "Initial lock attempt for shard={}" , shard ) ; final Lock lock = locks . get ( shard ) ; lock . acquire ( ) . get ( ) ; if ( lock . isHeld ( ) && ++ locksObtained >= maxLocksToPrefetch ) { break ; } } catch ( InterruptedException ex ) { log . warn ( "Thread exception while acquiring initial locks: " + ex . getMessage ( ) , ex ) ; } catch ( ExecutionException ex ) { log . error ( "Problem acquiring lock " + shard + " " + ex . getCause ( ) . getMessage ( ) , ex . getCause ( ) ) ; } } log . info ( "Finished pre-fetching zookeeper locks" ) ; } }


Input: Unsafe method for testing . [CODESPLIT] @ VisibleForTesting void waitForQuiesceUnsafe ( ) { while ( lockTaskCount . get ( ) > 0 ) { log . trace ( "Waiting for quiesce" ) ; try { Thread . sleep ( 100 ) ; } catch ( InterruptedException ignore ) { } } // this time out needs to be longer than ZK_LOCK_TIMEOUT for valid tests. try { Thread . sleep ( 2000 ) ; } catch ( InterruptedException ignore ) { } }


Input: Given the encoded slot key returns the java object of it . For the valid slot keys this function is inverse of { @link #toString () } . @return decoded { @link SlotKey } <code > null< / code > if it s an invalid slotkey . [CODESPLIT] public static SlotKey parse ( String string ) { String [ ] tokens = string . split ( "," ) ; if ( tokens . length != 3 ) { return null ; } Granularity granularity = Granularity . fromString ( tokens [ 0 ] ) ; if ( granularity == null ) { return null ; } try { int slot = Integer . parseInt ( tokens [ 1 ] ) ; int shard = Integer . parseInt ( tokens [ 2 ] ) ; return of ( granularity , slot , shard ) ; } catch ( IllegalArgumentException e ) { return null ; } }


Input: This method creates a collection of slot keys that are within the same timespan of the current slot key but of a finer granularity . For example a slot key of granularity { @link Granularity#MIN_20 MIN_20 } will have four child keys each of granularity { @link Granularity#MIN_5 MIN_5 } . <p > [CODESPLIT] public Collection < SlotKey > getChildrenKeys ( ) { if ( granularity == Granularity . FULL ) { return ImmutableList . of ( ) ; } List < SlotKey > result = new ArrayList < SlotKey > ( ) ; Granularity finer ; try { finer = granularity . finer ( ) ; } catch ( GranularityException e ) { throw new AssertionError ( "Should not occur." ) ; } int factor = finer . numSlots ( ) / granularity . numSlots ( ) ; for ( int i = 0 ; i < factor ; i ++ ) { int childSlot = slot * factor + i ; SlotKey child = SlotKey . of ( finer , childSlot , shard ) ; result . add ( child ) ; result . addAll ( child . getChildrenKeys ( ) ) ; } return result ; }


Input: Same as the method { @link #getChildrenKeys () } except this method returns only the children corresponding to the destination granularity . [CODESPLIT] public Collection < SlotKey > getChildrenKeys ( Granularity destGranularity ) { if ( ! getGranularity ( ) . isCoarser ( destGranularity ) ) { throw new IllegalArgumentException ( String . format ( "Current granularity [%s] must be coarser than the destination granularity [%s]" , getGranularity ( ) , destGranularity ) ) ; } List < SlotKey > result = new ArrayList < SlotKey > ( ) ; for ( SlotKey slotKey : this . getChildrenKeys ( ) ) { if ( slotKey . getGranularity ( ) . equals ( destGranularity ) ) { result . add ( slotKey ) ; } } return Collections . unmodifiableList ( result ) ; }


Input: This method would extrapolate a given slotkey to a corresponding parent slotkey of a given ( higher ) granularity . [CODESPLIT] public SlotKey extrapolate ( Granularity destGranularity ) { if ( destGranularity . equals ( this . getGranularity ( ) ) ) { return this ; } if ( ! destGranularity . isCoarser ( getGranularity ( ) ) ) { throw new IllegalArgumentException ( "Destination granularity must be coarser than the current granularity" ) ; } int factor = getGranularity ( ) . numSlots ( ) / destGranularity . numSlots ( ) ; int parentSlot = getSlot ( ) / factor ; return SlotKey . of ( destGranularity , parentSlot , getShard ( ) ) ; }


Input: Set the threadpool name . Used to generate metric names and thread names . [CODESPLIT] public ThreadPoolBuilder withName ( String name ) { // ensure we've got a spot to put the thread id. if ( ! name . contains ( "%d" ) ) { name = name + "-%d" ; } nameMap . putIfAbsent ( name , new AtomicInteger ( 0 ) ) ; int id = nameMap . get ( name ) . incrementAndGet ( ) ; this . poolName = String . format ( name , id ) ; if ( id > 1 ) { this . threadNameFormat = name . replace ( "%d" , id + "-%d" ) ; } else { this . threadNameFormat = name ; } return this ; }


Input: our own stuff . [CODESPLIT] @ Override public void handle ( ChannelHandlerContext ctx , FullHttpRequest request ) { Tracker . getInstance ( ) . track ( request ) ; requestCount . inc ( ) ; final Timer . Context timerContext = handlerTimer . time ( ) ; long ingestTime = clock . now ( ) . getMillis ( ) ; // this is all JSON. String body = null ; try { String submitterTenantId = request . headers ( ) . get ( HttpMetricsIngestionServer . TENANT_ID_HEADER ) ; body = request . content ( ) . toString ( Constants . DEFAULT_CHARSET ) ; List < AggregatedPayload > bundleList = createBundleList ( body ) ; if ( bundleList . size ( ) > 0 ) { // has aggregated metric bundle in body // convert and add metric bundle to MetricsCollection if valid MetricsCollection collection = new MetricsCollection ( ) ; List < ErrorResponse . ErrorData > errors = new ArrayList < ErrorResponse . ErrorData > ( ) ; // for each metric bundle int delayedMetricsCount = 0 ; int metricsCount = 0 ; for ( AggregatedPayload bundle : bundleList ) { // validate, convert, and add to collection List < ErrorResponse . ErrorData > bundleValidationErrors = bundle . getValidationErrors ( ) ; if ( bundleValidationErrors . isEmpty ( ) ) { // no validation error, add to collection collection . add ( PreaggregateConversions . buildMetricsCollection ( bundle ) ) ; } else { // failed validation, add to error errors . addAll ( bundleValidationErrors ) ; } if ( bundle . hasDelayedMetrics ( ingestTime ) ) { Tracker . getInstance ( ) . trackDelayedAggregatedMetricsTenant ( bundle . getTenantId ( ) , bundle . getTimestamp ( ) , bundle . getDelayTime ( ingestTime ) , bundle . getAllMetricNames ( ) ) ; bundle . markDelayMetricsReceived ( ingestTime ) ; delayedMetricsCount += bundle . getAllMetricNames ( ) . size ( ) ; } else { metricsCount += bundle . getAllMetricNames ( ) . size ( ) ; } } // if has validation errors and no valid metrics if ( ! errors . isEmpty ( ) && collection . size ( ) == 0 ) { // return BAD_REQUEST and error DefaultHandler . sendErrorResponse ( ctx , request , errors , HttpResponseStatus . BAD_REQUEST ) ; return ; } // process valid metrics in collection ListenableFuture < List < Boolean > > futures = processor . apply ( collection ) ; List < Boolean > persisteds = futures . get ( timeout . getValue ( ) , timeout . getUnit ( ) ) ; for ( Boolean persisted : persisteds ) { if ( ! persisted ) { DefaultHandler . sendErrorResponse ( ctx , request , "Internal error persisting data" , HttpResponseStatus . INTERNAL_SERVER_ERROR ) ; return ; } } recordPerTenantMetrics ( submitterTenantId , metricsCount , delayedMetricsCount ) ; // return OK or MULTI_STATUS response depending if there were validation errors if ( errors . isEmpty ( ) ) { // no validation error, response OK DefaultHandler . sendResponse ( ctx , request , null , HttpResponseStatus . OK ) ; return ; } else { // has some validation errors, response MULTI_STATUS DefaultHandler . sendErrorResponse ( ctx , request , errors , HttpResponseStatus . MULTI_STATUS ) ; return ; } } else { // no aggregated metric bundles in body, response OK DefaultHandler . sendResponse ( ctx , request , "No valid metrics" , HttpResponseStatus . BAD_REQUEST ) ; return ; } } catch ( JsonParseException ex ) { log . debug ( String . format ( "BAD JSON: %s" , body ) ) ; DefaultHandler . sendErrorResponse ( ctx , request , ex . getMessage ( ) , HttpResponseStatus . BAD_REQUEST ) ; } catch ( InvalidDataException ex ) { log . debug ( String . format ( "Invalid request body: %s" , body ) ) ; DefaultHandler . sendErrorResponse ( ctx , request , ex . getMessage ( ) , HttpResponseStatus . BAD_REQUEST ) ; } catch ( TimeoutException ex ) { DefaultHandler . sendErrorResponse ( ctx , request , "Timed out persisting metrics" , HttpResponseStatus . ACCEPTED ) ; } catch ( Exception ex ) { log . debug ( String . format ( "Exception processing: %s" , body ) ) ; log . error ( "Other exception while trying to parse content" , ex ) ; DefaultHandler . sendErrorResponse ( ctx , request , "Internal error saving data" , HttpResponseStatus . INTERNAL_SERVER_ERROR ) ; } finally { timerContext . stop ( ) ; requestCount . dec ( ) ; } }


Input:
For a given metricIndex and docCount classifies the data with respect to baseLevel and stores it accordingly . [CODESPLIT] public void add ( String metricIndex , long docCount ) { final String [ ] tokens = metricIndex . split ( METRIC_TOKEN_SEPARATOR_REGEX ) ; /**
         *
         * For the ES response shown in class description, for a baseLevel of 2,
         * the data is classified as follows
         *
         * metricNamesWithNextLevelSet    -> {foo.bar.baz}   (Metric Names which are at base + 1 level and also have a subsequent level.)
         *
         * For count map, data is in the form {metricIndex, (actualDocCount, childrenTotalDocCount)}
         *
         * metricNameBaseLevelMap   -> {foo.bar.baz -> (2, 1)}     (all indexes which are of same length as baseLevel)
         *
         */ switch ( tokens . length - baseLevel ) { case 1 : if ( baseLevel > 0 ) { metricNamesWithNextLevelSet . add ( metricIndex . substring ( 0 , metricIndex . lastIndexOf ( "." ) ) ) ; } else { metricNamesWithNextLevelSet . add ( metricIndex . substring ( 0 , metricIndex . indexOf ( "." ) ) ) ; } //For foo.bar.baz, baseLevel=3 we update children doc count of foo.bar.baz addChildrenDocCount ( metricNameBaseLevelMap , metricIndex . substring ( 0 , metricIndex . lastIndexOf ( "." ) ) , docCount ) ; break ; case 0 : setActualDocCount ( metricNameBaseLevelMap , metricIndex , docCount ) ; break ; default : break ; } }


Input: Compares actualDocCount and total docCount of its immediate children of an index to determine if the metric index is a complete metric name or not . [CODESPLIT] private Set < String > getCompleteMetricNames ( Map < String , MetricIndexDocCount > metricIndexMap ) { Set < String > completeMetricNames = new HashSet < String > ( ) ; for ( Map . Entry < String , MetricIndexDocCount > entry : metricIndexMap . entrySet ( ) ) { MetricIndexDocCount metricIndexDocCount = entry . getValue ( ) ; if ( metricIndexDocCount != null ) { //if total doc count is greater than its children docs, its a complete metric name if ( metricIndexDocCount . actualDocCount > 0 && metricIndexDocCount . actualDocCount > metricIndexDocCount . childrenTotalDocCount ) { completeMetricNames . add ( entry . getKey ( ) ) ; } } } return Collections . unmodifiableSet ( completeMetricNames ) ; }


Input: Side effect : mark dirty slots as clean [CODESPLIT] protected Map < Granularity , Map < Integer , UpdateStamp > > getDirtySlotsToPersist ( int shard ) { Map < Granularity , Map < Integer , UpdateStamp > > slotTimes = new HashMap < Granularity , Map < Integer , UpdateStamp > > ( ) ; int numUpdates = 0 ; for ( Granularity gran : Granularity . rollupGranularities ( ) ) { Map < Integer , UpdateStamp > dirty = getSlotStateManager ( shard , gran ) . getDirtySlotStampsAndMarkClean ( ) ; slotTimes . put ( gran , dirty ) ; if ( dirty . size ( ) > 0 ) { numUpdates += dirty . size ( ) ; } } if ( numUpdates > 0 ) { // for updates that come by way of scribe, you'll typically see 5 as the number of updates (one for // each granularity).  On rollup slaves the situation is a bit different. You'll see only the slot // of the granularity just written to marked dirty (so 1). log . debug ( "Found {} dirty slots for shard {}" , new Object [ ] { numUpdates , shard } ) ; return slotTimes ; } return null ; }


Input: This method return list of tokens with their parents for a current Discovery object . [CODESPLIT] public static List < Token > getTokens ( Locator locator ) { if ( StringUtils . isEmpty ( locator . getMetricName ( ) ) || StringUtils . isEmpty ( locator . getTenantId ( ) ) ) return new ArrayList <> ( ) ; String [ ] tokens = locator . getMetricName ( ) . split ( Locator . METRIC_TOKEN_SEPARATOR_REGEX ) ; return IntStream . range ( 0 , tokens . length ) . mapToObj ( index -> new Token ( locator , tokens , index ) ) . collect ( toList ( ) ) ; }


Input: Asynchronously insert a rolled up metric to the appropriate column family for a particular granularity [CODESPLIT] public ResultSetFuture putAsync ( Locator locator , long collectionTime , Rollup rollup , Granularity granularity , int ttl ) { Session session = DatastaxIO . getSession ( ) ; // we use batch statement here in case sub classes // override the addRollupToBatch() and provide // multiple statements BatchStatement batch = new BatchStatement ( ) ; addRollupToBatch ( batch , locator , rollup , collectionTime , granularity , ttl ) ; Collection < Statement > statements = batch . getStatements ( ) ; if ( statements . size ( ) == 1 ) { Statement oneStatement = statements . iterator ( ) . next ( ) ; return session . executeAsync ( oneStatement ) ; } else { LOG . debug ( String . format ( "Using BatchStatement for %d statements" , statements . size ( ) ) ) ; return session . executeAsync ( batch ) ; } }


Input: Fetch rollup objects for a { @link com . rackspacecloud . blueflood . types . Locator } from the specified column family and range . [CODESPLIT] protected < T extends Object > Table < Locator , Long , T > getRollupsForLocator ( final Locator locator , String columnFamily , Range range ) { return getValuesForLocators ( new ArrayList < Locator > ( ) { { add ( locator ) ; } } , columnFamily , range ) ; }


Input: Fetch values for a list of { @link com . rackspacecloud . blueflood . types . Locator } from the specified column family and range . [CODESPLIT] protected < T extends Object > Table < Locator , Long , T > getValuesForLocators ( final List < Locator > locators , String columnFamily , Range range ) { Table < Locator , Long , T > locatorTimestampRollup = HashBasedTable . create ( ) ; Map < Locator , List < ResultSetFuture > > resultSetFuturesMap = selectForLocatorListAndRange ( columnFamily , locators , range ) ; for ( Map . Entry < Locator , List < ResultSetFuture > > entry : resultSetFuturesMap . entrySet ( ) ) { Locator locator = entry . getKey ( ) ; List < ResultSetFuture > futures = entry . getValue ( ) ; Table < Locator , Long , T > result = toLocatorTimestampValue ( futures , locator , columnFamily , range ) ; locatorTimestampRollup . putAll ( result ) ; } return locatorTimestampRollup ; }


Input: Add a { @link com . datastax . driver . core . PreparedStatement } statement to the { @link com . datastax . driver . core . BatchStatement } to insert this Rollup object to metrics_preaggregated_ { granularity } column family [CODESPLIT] protected void addRollupToBatch ( BatchStatement batch , Locator locator , Rollup rollup , long collectionTime , Granularity granularity , int ttl ) { Statement statement = createStatement ( locator , collectionTime , rollup , granularity , ttl ) ; batch . add ( statement ) ; }


Input: Asynchronously execute select statements against the specified column family for a specific list of { @link com . rackspacecloud . blueflood . types . Locator } and { @link com . rackspacecloud . blueflood . types . Range } [CODESPLIT] protected Map < Locator , List < ResultSetFuture > > selectForLocatorListAndRange ( String columnFamilyName , List < Locator > locators , Range range ) { Map < Locator , List < ResultSetFuture > > locatorFuturesMap = new HashMap < Locator , List < ResultSetFuture > > ( ) ; for ( Locator locator : locators ) { List < ResultSetFuture > existing = locatorFuturesMap . get ( locator ) ; if ( existing == null ) { existing = new ArrayList < ResultSetFuture > ( ) ; locatorFuturesMap . put ( locator , existing ) ; } existing . addAll ( selectForLocatorAndRange ( columnFamilyName , locator , range ) ) ; } return locatorFuturesMap ; }


Input: Execute a select statement against the specified column family for a specific { @link com . rackspacecloud . blueflood . types . Locator } and { @link com . rackspacecloud . blueflood . types . Range } [CODESPLIT] protected List < ResultSetFuture > selectForLocatorAndRange ( String columnFamily , Locator locator , Range range ) { List < ResultSetFuture > resultsFutures = new ArrayList < ResultSetFuture > ( ) ; PreparedStatement statement = metricsCFPreparedStatements . cfNameToSelectStatement . get ( columnFamily ) ; resultsFutures . add ( session . executeAsync ( statement . bind ( locator . toString ( ) , range . getStart ( ) , range . getStop ( ) ) ) ) ; return resultsFutures ; }


Input: Give a { [CODESPLIT] public < T extends Object > Table < Locator , Long , T > toLocatorTimestampValue ( List < ResultSetFuture > futures , Locator locator , String columnFamily , Range range ) { Table < Locator , Long , T > locatorTimestampRollup = HashBasedTable . create ( ) ; for ( ResultSetFuture future : futures ) { try { List < Row > rows = future . getUninterruptibly ( ) . all ( ) ; // we only want to count the number of points we // get when we're querying the metrics_full if ( StringUtils . isNotEmpty ( columnFamily ) && columnFamily . equals ( CassandraModel . CF_METRICS_FULL_NAME ) ) { Instrumentation . getRawPointsIn5MinHistogram ( ) . update ( rows . size ( ) ) ; } for ( Row row : rows ) { String key = row . getString ( DMetricsCFPreparedStatements . KEY ) ; Locator loc = Locator . createLocatorFromDbKey ( key ) ; Long hash = row . getLong ( DMetricsCFPreparedStatements . COLUMN1 ) ; locatorTimestampRollup . put ( loc , hash , ( T ) fromByteBuffer ( row . getBytes ( DMetricsCFPreparedStatements . VALUE ) ) ) ; } } catch ( Exception ex ) { Instrumentation . markReadError ( ) ; LOG . error ( String . format ( "error reading metric for locator %s, column family '%s', range %s" , locator , columnFamily , range . toString ( ) ) , ex ) ; } } return locatorTimestampRollup ; }


Input: We need to derive ranges ( actual times ) from slots ( which are fixed integers that wrap ) when we discover a late slot . These ranges can be derived from a reference point ( which is usually something like now ) . [CODESPLIT] public Range deriveRange ( int slot , long referenceMillis ) { // referenceMillis refers to the current time in reference to the range we want to generate from the supplied  // slot. This implies that the range we wish to return is before slot(reference).  allow for slot wrapping. referenceMillis = snapMillis ( referenceMillis ) ; int refSlot = slot ( referenceMillis ) ; int slotDiff = slot > refSlot ? ( numSlots ( ) - slot + refSlot ) : ( refSlot - slot ) ; long rangeStart = referenceMillis - slotDiff * milliseconds ( ) ; return new Range ( rangeStart , rangeStart + milliseconds ( ) - 1 ) ; }


Input: Return granularity that maps most closely to requested number of points using the algorithm specified in the { @code GET_BY_POINTS_SELECTION_ALGORITHM } config value . See { @link #granularityFromPointsInInterval ( String long long int String long ) } . [CODESPLIT] public static Granularity granularityFromPointsInInterval ( String tenantid , long from , long to , int points ) { return granularityFromPointsInInterval ( tenantid , from , to , points , GET_BY_POINTS_SELECTION_ALGORITHM , GET_BY_POINTS_ASSUME_INTERVAL , DEFAULT_TTL_COMPARISON_SOURCE ) ; }


Input: { @code ttlComparisonClock } defaults to { @link #DEFAULT_TTL_COMPARISON_SOURCE } . [CODESPLIT] public static Granularity granularityFromPointsInInterval ( String tenantid , long from , long to , int points , String algorithm , long assumedIntervalMillis ) { return granularityFromPointsInInterval ( tenantid , from , to , points , algorithm , assumedIntervalMillis , DEFAULT_TTL_COMPARISON_SOURCE ) ; }


Input: Return granularity that maps most closely to requested number of points based on provided selection algorithm [CODESPLIT] public static Granularity granularityFromPointsInInterval ( String tenantid , long from , long to , int points , String algorithm , long assumedIntervalMillis , Clock ttlComparisonClock ) { if ( from >= to ) { throw new RuntimeException ( "Invalid interval specified for fromPointsInInterval" ) ; } double requestedDuration = to - from ; if ( algorithm . startsWith ( "GEOMETRIC" ) ) return granularityFromPointsGeometric ( tenantid , from , to , requestedDuration , points , assumedIntervalMillis , ttlComparisonClock ) ; else if ( algorithm . startsWith ( "LINEAR" ) ) return granularityFromPointsLinear ( requestedDuration , points , assumedIntervalMillis ) ; else if ( algorithm . startsWith ( "LESSTHANEQUAL" ) ) return granularityFromPointsLessThanEqual ( requestedDuration , points , assumedIntervalMillis ) ; return granularityFromPointsGeometric ( tenantid , from , to , requestedDuration , points , assumedIntervalMillis , ttlComparisonClock ) ; }


Input: Find the granularity in the interval that will yield a number of data points that are closest to the requested points but < = requested points . [CODESPLIT] private static Granularity granularityFromPointsLessThanEqual ( double requestedDuration , int points , long assumedIntervalMillis ) { Granularity gran = granularityFromPointsLinear ( requestedDuration , points , assumedIntervalMillis ) ; if ( requestedDuration / gran . milliseconds ( ) > points ) { try { gran = gran . coarser ( ) ; } catch ( GranularityException e ) { /* do nothing, already at 1440m */ } } return gran ; }


Input: Find the granularity in the interval that will yield a number of data points that are close to $points in terms of linear distance . [CODESPLIT] private static Granularity granularityFromPointsLinear ( double requestedDuration , int points , long assumedIntervalMillis ) { int closest = Integer . MAX_VALUE ; int diff = 0 ; Granularity gran = null ; for ( Granularity g : Granularity . granularities ( ) ) { if ( g == Granularity . FULL ) diff = ( int ) Math . abs ( points - ( requestedDuration / assumedIntervalMillis ) ) ; else diff = ( int ) Math . abs ( points - ( requestedDuration / g . milliseconds ( ) ) ) ; if ( diff < closest ) { closest = diff ; gran = g ; } else { break ; } } return gran ; }


Input: set the server time in millis . [CODESPLIT] public synchronized void setServerTime ( long millis ) { log . info ( "Manually setting server time to {}  {}" , millis , new java . util . Date ( millis ) ) ; context . setCurrentTimeMillis ( millis ) ; }


Input: Add a shard to be managed ( via JMX ) [CODESPLIT] public void addShard ( Integer shard ) { if ( ! shardStateManager . getManagedShards ( ) . contains ( shard ) ) context . addShard ( shard ) ; }


Input: Remove a shard from being managed ( via JMX ) [CODESPLIT] public void removeShard ( Integer shard ) { if ( shardStateManager . getManagedShards ( ) . contains ( shard ) ) context . removeShard ( shard ) ; }


Input: For all of these { @link Locator } s get a unique list of { @link Token } s which are not in token cache [CODESPLIT] static Set < Token > getUniqueTokens ( final List < Locator > locators ) { return Token . getUniqueTokens ( locators . stream ( ) ) . filter ( token -> ! TokenCache . getInstance ( ) . isTokenCurrent ( token ) ) . collect ( toSet ( ) ) ; }


Input: Get all { @link Locator } s corresponding to the metrics which are not current . [CODESPLIT] static List < Locator > getLocators ( final List < List < IMetric > > input ) { //converting list of lists of metrics to flat list of locators that are not current. return input . stream ( ) . flatMap ( List :: stream ) . map ( IMetric :: getLocator ) . filter ( locator -> ! LocatorCache . getInstance ( ) . isLocatorCurrentInTokenDiscoveryLayer ( locator ) ) . collect ( toList ( ) ) ; }


Input: For a given batch of metrics insert unique tokens using { @link TokenDiscoveryIO } . [CODESPLIT] public ListenableFuture < Boolean > processTokens ( final List < List < IMetric > > input ) { return getThreadPool ( ) . submit ( ( ) -> { boolean success = true ; List < Locator > locators = getLocators ( input ) ; List < Token > tokens = new ArrayList <> ( ) ; tokens . addAll ( getUniqueTokens ( locators ) ) ; if ( tokens . size ( ) > 0 ) { for ( TokenDiscoveryIO io : tokenDiscoveryIOs ) { try { io . insertDiscovery ( tokens ) ; } catch ( Exception ex ) { getLogger ( ) . error ( ex . getMessage ( ) , ex ) ; writeErrorMeters . get ( io . getClass ( ) ) . mark ( ) ; success = false ; } } } if ( success && tokens . size ( ) > 0 ) { tokens . stream ( ) . filter ( token -> ! token . isLeaf ( ) ) //do not cache leaf nodes . forEach ( token -> { //updating token cache TokenCache . getInstance ( ) . setTokenCurrent ( token ) ; } ) ; locators . stream ( ) . forEach ( locator -> { //updating locator cache LocatorCache . getInstance ( ) . setLocatorCurrentInTokenDiscoveryLayer ( locator ) ; } ) ; } return success ; } ) ; }


Input: Given a { [CODESPLIT] public static void instrument ( final ThreadPoolExecutor executor , String threadPoolName ) { MetricRegistry registry = Metrics . getRegistry ( ) ; registry . register ( name ( threadPoolName , "queue-size" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return executor . getQueue ( ) . size ( ) ; } } ) ; registry . register ( name ( threadPoolName , "queue-max" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return executor . getQueue ( ) . size ( ) + executor . getQueue ( ) . remainingCapacity ( ) ; } } ) ; registry . register ( name ( threadPoolName , "threadpool-active" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return executor . getActiveCount ( ) ; } } ) ; registry . register ( name ( threadPoolName , "threadpool-max" ) , new Gauge < Integer > ( ) { @ Override public Integer getValue ( ) { return executor . getMaximumPoolSize ( ) ; } } ) ; }


Input: Retrieves the set of unique Cassandra hosts from configuration file . If a single host appears multiple times in the configuration only one will be listed . [CODESPLIT] public Set < String > getUniqueHosts ( ) { Set < String > uniqueHosts = new HashSet < String > ( ) ; Collections . addAll ( uniqueHosts , config . getStringProperty ( CoreConfig . CASSANDRA_HOSTS ) . split ( "," ) ) ; return uniqueHosts ; }


Input: Retrieves a set of unique Cassandra hosts with Binary Transport protocol enabled from configuration file . This may or may not be different than { @link #getUniqueHosts () } [CODESPLIT] protected Set < String > getUniqueBinaryTransportHosts ( ) { Set < String > uniqueHosts = new HashSet < String > ( ) ; Collections . addAll ( uniqueHosts , config . getStringProperty ( CoreConfig . CASSANDRA_BINXPORT_HOSTS ) . split ( "," ) ) ; return uniqueHosts ; }


Input: Retrieves a set of unique Cassandra hosts with Binary Transport protocol enabled from configuration file as a set of { @link java . net . InetSocketAddress } objects . [CODESPLIT] public Set < InetSocketAddress > getUniqueBinaryTransportHostsAsInetSocketAddresses ( ) { Set < String > hosts = getUniqueBinaryTransportHosts ( ) ; Set < InetSocketAddress > inetAddresses = new HashSet < InetSocketAddress > ( ) ; for ( String host : hosts ) { String [ ] parts = host . split ( ":" ) ; InetSocketAddress inetSocketAddress ; if ( parts . length == 1 ) { inetSocketAddress = new InetSocketAddress ( parts [ 0 ] , config . getIntegerProperty ( CoreConfig . CASSANDRA_BINXPORT_PORT ) ) ; } else { inetSocketAddress = new InetSocketAddress ( parts [ 0 ] , Integer . parseInt ( parts [ 1 ] ) ) ; } inetAddresses . add ( inetSocketAddress ) ; } return inetAddresses ; }


Input: Calculates the number of max connections per Cassandra hosts [CODESPLIT] public int getMaxConnPerHost ( int numHosts ) { int maxConns = config . getIntegerProperty ( CoreConfig . MAX_CASSANDRA_CONNECTIONS ) ; return maxConns / numHosts + ( maxConns % numHosts == 0 ? 0 : 1 ) ; }


Input: Serialize without sum attribute . This is used by { @link GaugeSerDes } [CODESPLIT] protected void serializeRollupV1 ( BaseRollup baseRollup , CodedOutputStream protobufOut ) throws IOException { protobufOut . writeRawByte ( Constants . VERSION_1_ROLLUP ) ; serializeBaseRollupHelper ( baseRollup , protobufOut ) ; }


Input:
Builds ES query to grab tokens corresponding to the given query glob . For a given query foo . bar . * we would like to grab all the tokens with parent as foo . bar [CODESPLIT] private String getTokenQueryDslString ( String tenantId , String query ) { String [ ] queryTokens = query . split ( Locator . METRIC_TOKEN_SEPARATOR_REGEX ) ; String lastToken = queryTokens [ queryTokens . length - 1 ] ; /**
         * Builds parent part of the query for the given input query glob tokens.
         * For a given query foo.bar.*, parent part is foo.bar
         *
         * For example:
         *
         *  For query = foo.bar.*
         *          { "term": {  "parent": "foo.bar" }}
         *
         *  For query = foo.*.*
         *          { "regexp": {  "parent": "foo.[^.]+" }}
         *
         *  For query = foo.b*.*
         *          { "regexp": {  "parent": "foo.b[^.]*" }}
         */ String parentString = getParentString ( queryTokens ) ; String tenantIdTermQueryString = getTermQueryString ( ESFieldLabel . tenantId . name ( ) , tenantId ) ; List < String > mustNodes = new ArrayList <> ( ) ; mustNodes . add ( tenantIdTermQueryString ) ; mustNodes . add ( parentString ) ; // For example: if query=foo.bar.*, we can just get every token for the parent=foo.bar // but if query=foo.bar.b*, we want to add the token part of the query for "b*" if ( ! lastToken . equals ( "*" ) ) { String tokenQString ; GlobPattern pattern = new GlobPattern ( lastToken ) ; if ( pattern . hasWildcard ( ) ) { String compiledString = pattern . compiled ( ) . toString ( ) ; tokenQString = getRegexpQueryString ( ESFieldLabel . token . name ( ) , compiledString ) ; } else { tokenQString = getTermQueryString ( ESFieldLabel . token . name ( ) , lastToken ) ; } mustNodes . add ( tokenQString ) ; } String mustQueryString = getMustQueryString ( mustNodes ) ; String boolQueryString = getBoolQueryString ( mustQueryString ) ; // replace one '\' char with two '\\' boolQueryString = boolQueryString . replaceAll ( "\\\\" , "\\\\\\\\" ) ; return boolQueryString ; }


Input: For a given glob gives regex for { @code Locator . METRIC_TOKEN_SEPARATOR } separated tokens [CODESPLIT] protected String getRegexToHandleTokens ( GlobPattern globPattern ) { String [ ] queryRegexParts = globPattern . compiled ( ) . toString ( ) . split ( "\\\\." ) ; return Arrays . stream ( queryRegexParts ) . map ( this :: convertRegexToCaptureUptoNextToken ) . collect ( joining ( Locator . METRIC_TOKEN_SEPARATOR_REGEX ) ) ; }


Input:
If index () fails for whatever reason it always throws IOException because indexing failed for Elasticsearch . [CODESPLIT] public void index ( final String urlFormat , final String bulkString ) throws IOException { String tempUrl = String . format ( urlFormat , getNextBaseUrl ( ) ) ; HttpEntity entity = new NStringEntity ( bulkString , ContentType . APPLICATION_JSON ) ; int statusCode = 0 ; /*
        Here I am using Queue to keep a round-robin selection of next base url. If current base URL fails for
        whatever reason (with response == null), then in catch block, I am enqueueing the next base URL so that
        in next iteration call picks up the new URL. If URL works, queue will be empty and loop will break out.
        */ Queue < String > callQ = new LinkedList <> ( ) ; callQ . add ( tempUrl ) ; int callCount = 0 ; while ( ! callQ . isEmpty ( ) && callCount < MAX_CALL_COUNT ) { callCount ++ ; String url = callQ . remove ( ) ; logger . debug ( "Using url [{}]" , url ) ; HttpPost httpPost = new HttpPost ( url ) ; httpPost . setHeaders ( getHeaders ( ) ) ; httpPost . setEntity ( entity ) ; CloseableHttpResponse response = null ; try { logger . debug ( "ElasticsearchRestHelper.index Thread name in use: [{}]" , Thread . currentThread ( ) . getName ( ) ) ; response = closeableHttpClient . execute ( httpPost ) ; statusCode = response . getStatusLine ( ) . getStatusCode ( ) ; String str = EntityUtils . toString ( response . getEntity ( ) ) ; EntityUtils . consume ( response . getEntity ( ) ) ; if ( statusCode != HttpStatus . SC_OK && statusCode != HttpStatus . SC_CREATED ) { logger . error ( "index method failed with status code: {} and error: {}" , statusCode , str ) ; } } catch ( Exception e ) { if ( response == null ) { logger . error ( "index method failed with message: {}" , e . getMessage ( ) ) ; url = String . format ( urlFormat , getNextBaseUrl ( ) ) ; callQ . add ( url ) ; } else { logger . error ( "index method failed with status code: {} and exception message: {}" , statusCode , e . getMessage ( ) ) ; } } finally { if ( response != null ) { response . close ( ) ; } } } if ( statusCode != HttpStatus . SC_OK && statusCode != HttpStatus . SC_CREATED ) throw new IOException ( "Elasticsearch indexing failed with status code: [" + statusCode + "]" ) ; }


Input: iterate over all column families that store metrics . [CODESPLIT] public static Iterable < MetricColumnFamily > getMetricColumnFamilies ( ) { return new Iterable < MetricColumnFamily > ( ) { @ Override public Iterator < MetricColumnFamily > iterator ( ) { return new Iterator < MetricColumnFamily > ( ) { private int pos = 0 ; @ Override public boolean hasNext ( ) { return pos < METRIC_COLUMN_FAMILES . length ; } @ Override public MetricColumnFamily next ( ) { return METRIC_COLUMN_FAMILES [ pos ++ ] ; } @ Override public void remove ( ) { throw new NoSuchMethodError ( "Not implemented" ) ; } } ; } } ; }


Input: should be populated . [CODESPLIT] private static Map < String , Object > parseOptions ( String [ ] args ) { final GnuParser parser = new GnuParser ( ) ; final Map < String , Object > options = new HashMap < String , Object > ( ) ; try { final long now = System . currentTimeMillis ( ) ; CommandLine line = parser . parse ( cliOptions , args ) ; options . put ( SRC , line . getOptionValue ( SRC ) ) ; options . put ( DST , line . getOptionValue ( DST ) ) ; // default range is one year ago until now. options . put ( FROM , line . hasOption ( FROM ) ? parseDateTime ( line . getOptionValue ( FROM ) ) : now - ( 365L * 24L * 60L * 60L * 1000L ) ) ; options . put ( TO , line . hasOption ( TO ) ? parseDateTime ( line . getOptionValue ( TO ) ) : now ) ; options . put ( LIMIT , line . hasOption ( LIMIT ) ? Integer . parseInt ( line . getOptionValue ( LIMIT ) ) : Integer . MAX_VALUE ) ; options . put ( SKIP , line . hasOption ( SKIP ) ? Integer . parseInt ( line . getOptionValue ( SKIP ) ) : 0 ) ; options . put ( BATCH_SIZE , line . hasOption ( BATCH_SIZE ) ? Integer . parseInt ( line . getOptionValue ( BATCH_SIZE ) ) : 100 ) ; // create a mapping of all cf names -> cf. // then determine which column family to process. Map < String , ColumnFamily < Locator , Long > > nameToCf = new HashMap < String , ColumnFamily < Locator , Long > > ( ) { { for ( CassandraModel . MetricColumnFamily cf : CassandraModel . getMetricColumnFamilies ( ) ) { put ( cf . getName ( ) , cf ) ; } } } ; if ( nameToCf . get ( line . getOptionValue ( COLUMN_FAMILY ) ) == null ) { throw new ParseException ( "Invalid column family" ) ; } CassandraModel . MetricColumnFamily columnFamily = ( CassandraModel . MetricColumnFamily ) nameToCf . get ( line . getOptionValue ( COLUMN_FAMILY ) ) ; options . put ( COLUMN_FAMILY , columnFamily ) ; options . put ( TTL , line . hasOption ( TTL ) ? Integer . parseInt ( line . getOptionValue ( TTL ) ) : ( int ) ( 5 * columnFamily . getDefaultTTL ( ) . toSeconds ( ) ) ) ; options . put ( READ_THREADS , line . hasOption ( READ_THREADS ) ? Integer . parseInt ( line . getOptionValue ( READ_THREADS ) ) : 1 ) ; options . put ( WRITE_THREADS , line . hasOption ( WRITE_THREADS ) ? Integer . parseInt ( line . getOptionValue ( WRITE_THREADS ) ) : 1 ) ; options . put ( VERIFY , line . hasOption ( VERIFY ) ) ; options . put ( DISCOVER , line . hasOption ( DISCOVER ) ? NodeDiscoveryType . RING_DESCRIBE : NodeDiscoveryType . NONE ) ; options . put ( RATE , line . hasOption ( RATE ) ? Integer . parseInt ( line . getOptionValue ( RATE ) ) : 500 ) ; } catch ( ParseException ex ) { HelpFormatter helpFormatter = new HelpFormatter ( ) ; helpFormatter . printHelp ( "bf-migrate" , cliOptions ) ; System . exit ( - 1 ) ; } return options ; }


Input: Provides a way for the sub class to get a { @link java . nio . ByteBuffer } representation of a certain Rollup object . [CODESPLIT] @ Override protected ByteBuffer toByteBuffer ( Object value ) { if ( ! ( value instanceof BluefloodTimerRollup ) ) { throw new IllegalArgumentException ( "toByteBuffer(): expecting BluefloodTimerRollup class but got " + value . getClass ( ) . getSimpleName ( ) ) ; } BluefloodTimerRollup TimerRollup = ( BluefloodTimerRollup ) value ; return serDes . serialize ( TimerRollup ) ; }


Input: Inserts a collection of metrics to the metrics_preaggregated_full column family [CODESPLIT] @ Override public void insertMetrics ( Collection < IMetric > metrics ) throws IOException { insertMetrics ( metrics , Granularity . FULL ) ; }


Input: Inserts a collection of metrics to the correct column family based on the specified granularity [CODESPLIT] @ Override public void insertMetrics ( Collection < IMetric > metrics , Granularity granularity ) throws IOException { try { AstyanaxWriter . getInstance ( ) . insertMetrics ( metrics , CassandraModel . getPreaggregatedColumnFamily ( granularity ) , isRecordingDelayedMetrics , clock ) ; } catch ( ConnectionException ex ) { throw new IOException ( ex ) ; } }


Input: Fetches { @link com . rackspacecloud . blueflood . outputs . formats . MetricData } objects for the specified { @link com . rackspacecloud . blueflood . types . Locator } and { @link com . rackspacecloud . blueflood . types . Range } from the specified column family [CODESPLIT] @ Override public MetricData getDatapointsForRange ( final Locator locator , Range range , Granularity gran ) { return AstyanaxReader . getInstance ( ) . getDatapointsForRange ( locator , range , gran ) ; }


Input: Fetches { @link com . rackspacecloud . blueflood . outputs . formats . MetricData } objects for the specified list of { @link com . rackspacecloud . blueflood . types . Locator } and { @link com . rackspacecloud . blueflood . types . Range } from the specified column family [CODESPLIT] @ Override public Map < Locator , MetricData > getDatapointsForRange ( List < Locator > locators , Range range , Granularity gran ) { return AstyanaxReader . getInstance ( ) . getDatapointsForRange ( locators , range , gran ) ; }


Input: Fetches a { @link com . rackspacecloud . blueflood . types . Points } object for a particular locator and rollupType from the specified column family and range [CODESPLIT] @ Override public < T extends Rollup > Points < T > getDataToRollup ( final Locator locator , RollupType rollupType , Range range , String columnFamilyName ) throws IOException { ColumnFamily cf = CassandraModel . getColumnFamily ( columnFamilyName ) ; // a quick n dirty hack, this code will go away someday Granularity granularity = CassandraModel . getGranularity ( cf ) ; Class < ? extends Rollup > rollupClass = RollupType . classOf ( rollupType , granularity ) ; return AstyanaxReader . getInstance ( ) . getDataToRoll ( rollupClass , locator , range , cf ) ; }


Input: Inserts a raw metric ( not its rolled up value ) to the proper column family [CODESPLIT] public ResultSetFuture insertRawAsync ( IMetric metric ) { BoundStatement bound = getBoundStatementForMetric ( metric ) ; return session . executeAsync ( bound ) ; }


Input: Constructs a { @link BoundStatement } for a particular metric . The statement could be part of a BatchStatement or it could be executed as is . [CODESPLIT] public BoundStatement getBoundStatementForMetric ( IMetric metric ) { return metricsCFPreparedStatements . insertToMetricsBasicFullStatement . bind ( metric . getLocator ( ) . toString ( ) , metric . getCollectionTime ( ) , serDes . serialize ( metric . getMetricValue ( ) ) , metric . getTtlInSeconds ( ) ) ; }


Input: This methods gets locators to rollup a slot . [CODESPLIT] protected Set < Locator > getLocators ( RollupExecutionContext executionContext , boolean isReroll , Granularity delayedMetricsRerollGranularity , Granularity delayedMetricsStorageGranularity ) { Set < Locator > locators = new HashSet < Locator > ( ) ; //if delayed metric tracking is enabled, if its re-roll, if slot granularity is no coarser than // DELAYED_METRICS_REROLL_GRANULARITY, get delayed locators if ( RECORD_DELAYED_METRICS && isReroll && ! getGranularity ( ) . isCoarser ( delayedMetricsRerollGranularity ) ) { if ( getGranularity ( ) . isCoarser ( delayedMetricsStorageGranularity ) ) { // For example, if we are re-rolling a 60m slot, and we store delayed metrics at 20m, we need to // grab delayed metrics for 3 * 20m slots corresponding to the 60m slot. for ( SlotKey slotKey : parentSlotKey . getChildrenKeys ( delayedMetricsStorageGranularity ) ) { locators . addAll ( getDelayedLocators ( executionContext , slotKey ) ) ; } } else { locators = getDelayedLocators ( executionContext , parentSlotKey . extrapolate ( delayedMetricsStorageGranularity ) ) ; } } else { locators = getLocators ( executionContext ) ; } return locators ; }


Input: This method returns a list of { @link MetricName } s matching the given glob query . [CODESPLIT] public List < MetricName > getMetricNames ( final String tenant , final String query ) throws Exception { Timer . Context esMetricNamesQueryTimerCtx = esMetricNamesQueryTimer . time ( ) ; String response ; try { String regexString = regexToGrabCurrentAndNextLevel ( query ) ; // replace one '\' char with two '\\' regexString = regexString . replaceAll ( "\\\\" , "\\\\\\\\" ) ; response = getMetricNamesFromES ( tenant , regexString ) ; } finally { esMetricNamesQueryTimerCtx . stop ( ) ; } // For example, if query = foo.bar.*, base level is 3 which is equal to the number of tokens in the query. int baseLevel = getTotalTokens ( query ) ; MetricIndexData metricIndexData = getMetricIndexData ( response , baseLevel ) ; List < MetricName > metricNames = new ArrayList <> ( ) ; //Metric Names matching query which have next level metricNames . addAll ( metricIndexData . getMetricNamesWithNextLevel ( ) . stream ( ) . map ( x -> new MetricName ( x , false ) ) . collect ( toSet ( ) ) ) ; //complete metric names matching query metricNames . addAll ( metricIndexData . getCompleteMetricNamesAtBaseLevel ( ) . stream ( ) . map ( x -> new MetricName ( x , true ) ) . collect ( toSet ( ) ) ) ; return metricNames ; }


Input: Performs terms aggregation by metric_name which returns doc_count by metric_name index that matches the given regex . [CODESPLIT] private String getMetricNamesFromES ( final String tenant , final String regexMetricName ) throws IOException { String metricNamesFromElasticsearchQueryString = String . format ( queryToFetchMetricNamesFromElasticsearchFormat , tenant , regexMetricName , regexMetricName ) ; return elasticsearchRestHelper . fetchDocuments ( ELASTICSEARCH_INDEX_NAME_READ , ELASTICSEARCH_DOCUMENT_TYPE , tenant , metricNamesFromElasticsearchQueryString ) ; }


Input: Returns regex which could grab metric names from current level to the next level for a given query . [CODESPLIT] protected String regexToGrabCurrentAndNextLevel ( final String query ) { if ( StringUtils . isEmpty ( query ) ) { throw new IllegalArgumentException ( "Query(glob) string cannot be null/empty" ) ; } String queryRegex = getRegex ( query ) ; int totalQueryTokens = getTotalTokens ( query ) ; if ( totalQueryTokens == 1 ) { // get metric names which matches the given query and have a next level, // Ex: For metric foo.bar.baz.qux, if query=*, we should get foo.bar. We are not // grabbing 0 level as it will give back bar, baz, qux because of the way data is structured. String baseRegex = convertRegexToCaptureUptoNextToken ( queryRegex ) ; return baseRegex + METRIC_TOKEN_SEPARATOR_REGEX + REGEX_TO_GRAB_SINGLE_TOKEN ; } else { String [ ] queryRegexParts = queryRegex . split ( "\\\\." ) ; String queryRegexUptoPrevLevel = StringUtils . join ( queryRegexParts , METRIC_TOKEN_SEPARATOR_REGEX , 0 , totalQueryTokens - 1 ) ; String baseRegex = convertRegexToCaptureUptoNextToken ( queryRegexUptoPrevLevel ) ; String queryRegexLastLevel = queryRegexParts [ totalQueryTokens - 1 ] ; String lastTokenRegex = convertRegexToCaptureUptoNextToken ( queryRegexLastLevel ) ; // Ex: For metric foo.bar.baz.qux.xxx, if query=foo.bar.b*, get foo.bar.baz, foo.bar.baz.qux // In this case baseRegex = "foo.bar", lastTokenRegex = "b[^.]*"' and the final // regex is foo\.bar\.b[^.]*(\.[^.]*){0,1} return baseRegex + METRIC_TOKEN_SEPARATOR_REGEX + lastTokenRegex + "(" + METRIC_TOKEN_SEPARATOR_REGEX + REGEX_TO_GRAB_SINGLE_TOKEN + ")" + "{0,1}" ; } }


Input: Start background storage management and uploading tasks . [CODESPLIT] public synchronized void start ( ) { if ( uploaderThread != null ) { throw new RuntimeException ( "StorageManager is already started" ) ; } fileUploader = new DoneFileUploader ( ) ; uploaderThread = new Thread ( fileUploader , "StorageManager uploader" ) ; uploaderThread . start ( ) ; }


Input: Stop background storage management . [CODESPLIT] public synchronized void stop ( ) throws IOException { if ( uploaderThread == null ) { throw new RuntimeException ( "Not running" ) ; } uploaderThread . interrupt ( ) ; uploaderThread = null ; fileUploader . shutdown ( ) ; }


Input: merge simple numbers with this rollup . [CODESPLIT] protected void computeFromSimpleMetrics ( Points < SimpleNumber > input ) throws IOException { super . computeFromSimpleMetrics ( input ) ; if ( input . isEmpty ( ) ) { return ; } Map < Long , Points . Point < SimpleNumber > > points = input . getPoints ( ) ; for ( Map . Entry < Long , Points . Point < SimpleNumber > > item : points . entrySet ( ) ) { SimpleNumber numericMetric = item . getValue ( ) . getData ( ) ; sum += numericMetric . getValue ( ) . doubleValue ( ) ; } }


Input: merge rollups into this rollup . [CODESPLIT] protected void computeFromRollups ( Points < BasicRollup > input ) throws IOException { computeFromRollupsHelper ( input ) ; if ( input . isEmpty ( ) ) { return ; } // See this and get mind blown: // http://stackoverflow.com/questions/18907262/bounded-wildcard-related-compiler-error Map < Long , ? extends Points . Point < ? extends BasicRollup > > points = input . getPoints ( ) ; for ( Map . Entry < Long , ? extends Points . Point < ? extends BasicRollup > > item : points . entrySet ( ) ) { BasicRollup rollup = item . getValue ( ) . getData ( ) ; if ( ! ( rollup instanceof BasicRollup ) ) { throw new IOException ( "Cannot create BasicRollup from type " + rollup . getClass ( ) . getName ( ) ) ; } BasicRollup basicRollup = ( BasicRollup ) rollup ; sum += basicRollup . getSum ( ) ; } }


Input: Checks if Locator is recently inserted in the batch layer [CODESPLIT] public synchronized boolean isLocatorCurrentInBatchLayer ( Locator loc ) { LocatorCacheEntry entry = insertedLocators . getIfPresent ( loc . toString ( ) ) ; return entry != null && entry . isBatchCurrent ( ) ; }


Input: Checks if Locator is recently inserted in the discovery layer [CODESPLIT] public synchronized boolean isLocatorCurrentInDiscoveryLayer ( Locator loc ) { LocatorCacheEntry entry = insertedLocators . getIfPresent ( loc . toString ( ) ) ; return entry != null && entry . isDiscoveryCurrent ( ) ; }


Input: Checks if Locator is recently inserted in the token discovery layer [CODESPLIT] public synchronized boolean isLocatorCurrentInTokenDiscoveryLayer ( Locator loc ) { LocatorCacheEntry entry = insertedLocators . getIfPresent ( loc . toString ( ) ) ; return entry != null && entry . isTokenDiscoveryCurrent ( ) ; }


Input: Check if the delayed locator is recently inserted for a given slot [CODESPLIT] public synchronized boolean isDelayedLocatorForASlotCurrent ( int slot , Locator locator ) { return insertedDelayedLocators . getIfPresent ( getLocatorSlotKey ( slot , locator ) ) != null ; }


Input: Marks the delayed locator as recently inserted for a given slot [CODESPLIT] public synchronized void setDelayedLocatorForASlotCurrent ( int slot , Locator locator ) { insertedDelayedLocators . put ( getLocatorSlotKey ( slot , locator ) , Boolean . TRUE ) ; }


Input: Provides a way for the sub class to get a { @link java . nio . ByteBuffer } representation of a certain Rollup object . [CODESPLIT] @ Override protected ByteBuffer toByteBuffer ( Object value ) { if ( ! ( value instanceof BluefloodSetRollup ) ) { throw new IllegalArgumentException ( "toByteBuffer(): expecting BluefloodSetRollup class but got " + value . getClass ( ) . getSimpleName ( ) ) ; } BluefloodSetRollup setRollup = ( BluefloodSetRollup ) value ; return serDes . serialize ( setRollup ) ; }


Input: given a start and stop time return an iterator over ranges in * this * granularity that should be rolled up into single points in the next coarser granularity . [CODESPLIT] public static Iterable < Range > getRangesToRollup ( Granularity g , final long startMillis , final long stopMillis ) throws GranularityException { final long snappedStartMillis = g . coarser ( ) . snapMillis ( startMillis ) ; final long snappedStopMillis = g . coarser ( ) . snapMillis ( stopMillis + g . coarser ( ) . milliseconds ( ) ) ; return new IntervalRangeIterator ( g , snappedStartMillis , snappedStopMillis ) ; }