List of usage examples for org.joda.time Interval getStartMillis
public long getStartMillis()
From source file:org.apache.druid.segment.realtime.plumber.Sink.java
License:Apache License
public Sink(Interval interval, DataSchema schema, ShardSpec shardSpec, String version, int maxRowsInMemory, long maxBytesInMemory, boolean reportParseExceptions, String dedupColumn) { this.schema = schema; this.shardSpec = shardSpec; this.interval = interval; this.version = version; this.maxRowsInMemory = maxRowsInMemory; this.maxBytesInMemory = maxBytesInMemory; this.reportParseExceptions = reportParseExceptions; this.dedupColumn = dedupColumn; makeNewCurrIndex(interval.getStartMillis(), schema); }
From source file:org.apache.druid.segment.realtime.plumber.Sink.java
License:Apache License
public Sink(Interval interval, DataSchema schema, ShardSpec shardSpec, String version, int maxRowsInMemory, long maxBytesInMemory, boolean reportParseExceptions, String dedupColumn, List<FireHydrant> hydrants) { this.schema = schema; this.shardSpec = shardSpec; this.interval = interval; this.version = version; this.maxRowsInMemory = maxRowsInMemory; this.maxBytesInMemory = maxBytesInMemory; this.reportParseExceptions = reportParseExceptions; this.dedupColumn = dedupColumn; int maxCount = -1; for (int i = 0; i < hydrants.size(); ++i) { final FireHydrant hydrant = hydrants.get(i); if (hydrant.getCount() <= maxCount) { throw new ISE("hydrant[%s] not the right count[%s]", hydrant, i); }/*from w ww . ja v a 2s . c o m*/ maxCount = hydrant.getCount(); ReferenceCountingSegment segment = hydrant.getIncrementedSegment(); try { numRowsExcludingCurrIndex.addAndGet(segment.asQueryableIndex().getNumRows()); } finally { segment.decrement(); } } this.hydrants.addAll(hydrants); makeNewCurrIndex(interval.getStartMillis(), schema); }
From source file:org.apache.druid.server.coordinator.CostBalancerStrategy.java
License:Apache License
/** * This defines the unnormalized cost function between two segments. * * See https://github.com/apache/incubator-druid/pull/2972 for more details about the cost function. * * intervalCost: segments close together are more likely to be queried together * * multiplier: if two segments belong to the same data source, they are more likely to be involved * in the same queries/*from w w w. j av a2 s . c om*/ * * @param segmentA The first DataSegment. * @param segmentB The second DataSegment. * * @return the joint cost of placing the two DataSegments together on one node. */ public static double computeJointSegmentsCost(final DataSegment segmentA, final DataSegment segmentB) { final Interval intervalA = segmentA.getInterval(); final Interval intervalB = segmentB.getInterval(); final double t0 = intervalA.getStartMillis(); final double t1 = (intervalA.getEndMillis() - t0) / MILLIS_FACTOR; final double start = (intervalB.getStartMillis() - t0) / MILLIS_FACTOR; final double end = (intervalB.getEndMillis() - t0) / MILLIS_FACTOR; // constant cost-multiplier for segments of the same datsource final double multiplier = segmentA.getDataSource().equals(segmentB.getDataSource()) ? 2.0 : 1.0; return INV_LAMBDA_SQUARE * intervalCost(t1, start, end) * multiplier; }
From source file:org.apache.druid.server.coordinator.rules.PeriodDropRule.java
License:Apache License
@Override public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) { final Interval currInterval = new Interval(period, referenceTimestamp); if (includeFuture) { return currInterval.getStartMillis() <= theInterval.getStartMillis(); } else {/*from w w w . j a v a2 s.c o m*/ return currInterval.contains(theInterval); } }
From source file:org.apache.druid.server.coordinator.rules.Rules.java
License:Apache License
public static boolean eligibleForLoad(Period period, Interval interval, DateTime referenceTimestamp, boolean includeFuture) { final Interval currInterval = new Interval(period, referenceTimestamp); if (includeFuture) { return currInterval.getStartMillis() < interval.getEndMillis(); } else {//from w ww . ja v a 2s . c o m return eligibleForLoad(currInterval, interval); } }
From source file:org.apache.druid.sql.calcite.expression.Expressions.java
License:Apache License
/** * Translates to a simple leaf filter, i.e. not an "expression" type filter. Note that the filter may still * reference expression virtual columns, if and only if "virtualColumnRegistry" is defined. * * @param plannerContext planner context * @param rowSignature input row signature * @param virtualColumnRegistry re-usable virtual column references, may be null if virtual columns aren't allowed * @param rexNode Calcite row expression */// www . j a v a 2s . c o m @Nullable private static DimFilter toSimpleLeafFilter(final PlannerContext plannerContext, final RowSignature rowSignature, @Nullable final VirtualColumnRegistry virtualColumnRegistry, final RexNode rexNode) { final SqlKind kind = rexNode.getKind(); if (kind == SqlKind.IS_TRUE || kind == SqlKind.IS_NOT_FALSE) { return toSimpleLeafFilter(plannerContext, rowSignature, virtualColumnRegistry, Iterables.getOnlyElement(((RexCall) rexNode).getOperands())); } else if (kind == SqlKind.IS_FALSE || kind == SqlKind.IS_NOT_TRUE) { return new NotDimFilter(toSimpleLeafFilter(plannerContext, rowSignature, virtualColumnRegistry, Iterables.getOnlyElement(((RexCall) rexNode).getOperands()))); } else if (kind == SqlKind.IS_NULL || kind == SqlKind.IS_NOT_NULL) { final RexNode operand = Iterables.getOnlyElement(((RexCall) rexNode).getOperands()); final DruidExpression druidExpression = toDruidExpression(plannerContext, rowSignature, operand); if (druidExpression == null) { return null; } final DimFilter equalFilter; if (druidExpression.isSimpleExtraction()) { equalFilter = new SelectorDimFilter(druidExpression.getSimpleExtraction().getColumn(), NullHandling.defaultStringValue(), druidExpression.getSimpleExtraction().getExtractionFn()); } else if (virtualColumnRegistry != null) { final VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, operand.getType().getSqlTypeName()); equalFilter = new SelectorDimFilter(virtualColumn.getOutputName(), NullHandling.defaultStringValue(), null); } else { return null; } return kind == SqlKind.IS_NOT_NULL ? new NotDimFilter(equalFilter) : equalFilter; } else if (kind == SqlKind.EQUALS || kind == SqlKind.NOT_EQUALS || kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL) { final List<RexNode> operands = ((RexCall) rexNode).getOperands(); Preconditions.checkState(operands.size() == 2, "WTF?! Expected 2 operands, got[%,d]", operands.size()); boolean flip = false; RexNode lhs = operands.get(0); RexNode rhs = operands.get(1); if (lhs.getKind() == SqlKind.LITERAL && rhs.getKind() != SqlKind.LITERAL) { // swap lhs, rhs RexNode x = lhs; lhs = rhs; rhs = x; flip = true; } // Flip operator, maybe. final SqlKind flippedKind; if (flip) { switch (kind) { case EQUALS: case NOT_EQUALS: flippedKind = kind; break; case GREATER_THAN: flippedKind = SqlKind.LESS_THAN; break; case GREATER_THAN_OR_EQUAL: flippedKind = SqlKind.LESS_THAN_OR_EQUAL; break; case LESS_THAN: flippedKind = SqlKind.GREATER_THAN; break; case LESS_THAN_OR_EQUAL: flippedKind = SqlKind.GREATER_THAN_OR_EQUAL; break; default: throw new ISE("WTF?! Kind[%s] not expected here", kind); } } else { flippedKind = kind; } // rhs must be a literal if (rhs.getKind() != SqlKind.LITERAL) { return null; } // Translate lhs to a DruidExpression. final DruidExpression lhsExpression = toDruidExpression(plannerContext, rowSignature, lhs); if (lhsExpression == null) { return null; } // Special handling for filters on FLOOR(__time TO granularity). final Granularity queryGranularity = toQueryGranularity(lhsExpression, plannerContext.getExprMacroTable()); if (queryGranularity != null) { // lhs is FLOOR(__time TO granularity); rhs must be a timestamp final long rhsMillis = Calcites.calciteDateTimeLiteralToJoda(rhs, plannerContext.getTimeZone()) .getMillis(); return buildTimeFloorFilter(ColumnHolder.TIME_COLUMN_NAME, queryGranularity, flippedKind, rhsMillis); } final String column; final ExtractionFn extractionFn; if (lhsExpression.isSimpleExtraction()) { column = lhsExpression.getSimpleExtraction().getColumn(); extractionFn = lhsExpression.getSimpleExtraction().getExtractionFn(); } else if (virtualColumnRegistry != null) { VirtualColumn virtualLhs = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, lhsExpression, lhs.getType().getSqlTypeName()); column = virtualLhs.getOutputName(); extractionFn = null; } else { return null; } if (column.equals(ColumnHolder.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) { // Check if we can strip the extractionFn and convert the filter to a direct filter on __time. // This allows potential conversion to query-level "intervals" later on, which is ideal for Druid queries. final Granularity granularity = ExtractionFns.toQueryGranularity(extractionFn); if (granularity != null) { // lhs is FLOOR(__time TO granularity); rhs must be a timestamp final long rhsMillis = Calcites.calciteDateTimeLiteralToJoda(rhs, plannerContext.getTimeZone()) .getMillis(); final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis)); // Is rhs aligned on granularity boundaries? final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis; // Create a BoundRefKey that strips the extractionFn and compares __time as a number. final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC); return getBoundTimeDimFilter(flippedKind, boundRefKey, rhsInterval, rhsAligned); } } final String val; final RexLiteral rhsLiteral = (RexLiteral) rhs; if (SqlTypeName.NUMERIC_TYPES.contains(rhsLiteral.getTypeName())) { val = String.valueOf(RexLiteral.value(rhsLiteral)); } else if (SqlTypeName.CHAR_TYPES.contains(rhsLiteral.getTypeName())) { val = String.valueOf(RexLiteral.stringValue(rhsLiteral)); } else if (SqlTypeName.TIMESTAMP == rhsLiteral.getTypeName() || SqlTypeName.DATE == rhsLiteral.getTypeName()) { val = String.valueOf(Calcites.calciteDateTimeLiteralToJoda(rhsLiteral, plannerContext.getTimeZone()) .getMillis()); } else { // Don't know how to filter on this kind of literal. return null; } // Numeric lhs needs a numeric comparison. final StringComparator comparator = Calcites .getStringComparatorForSqlTypeName(lhs.getType().getSqlTypeName()); final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator); final DimFilter filter; // Always use BoundDimFilters, to simplify filter optimization later (it helps to remember the comparator). switch (flippedKind) { case EQUALS: filter = Bounds.equalTo(boundRefKey, val); break; case NOT_EQUALS: filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val)); break; case GREATER_THAN: filter = Bounds.greaterThan(boundRefKey, val); break; case GREATER_THAN_OR_EQUAL: filter = Bounds.greaterThanOrEqualTo(boundRefKey, val); break; case LESS_THAN: filter = Bounds.lessThan(boundRefKey, val); break; case LESS_THAN_OR_EQUAL: filter = Bounds.lessThanOrEqualTo(boundRefKey, val); break; default: throw new IllegalStateException("WTF?! Shouldn't have got here..."); } return filter; } else if (rexNode instanceof RexCall) { final SqlOperator operator = ((RexCall) rexNode).getOperator(); final SqlOperatorConversion conversion = plannerContext.getOperatorTable() .lookupOperatorConversion(operator); if (conversion == null) { return null; } else { return conversion.toDruidFilter(plannerContext, rowSignature, virtualColumnRegistry, rexNode); } } else { return null; } }
From source file:org.apache.druid.sql.calcite.expression.Expressions.java
License:Apache License
/** * Build a filter for an expression like FLOOR(column TO granularity) [operator] rhsMillis *//*from w ww .jav a 2 s . c o m*/ private static DimFilter buildTimeFloorFilter(final String column, final Granularity granularity, final SqlKind operatorKind, final long rhsMillis) { final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC); final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis)); // Is rhs aligned on granularity boundaries? final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis; return getBoundTimeDimFilter(operatorKind, boundRefKey, rhsInterval, rhsAligned); }
From source file:org.apache.druid.sql.calcite.expression.Expressions.java
License:Apache License
private static DimFilter getBoundTimeDimFilter(SqlKind operatorKind, BoundRefKey boundRefKey, Interval interval, boolean isAligned) { switch (operatorKind) { case EQUALS://from w w w . ja v a 2s. c o m return isAligned ? Bounds.interval(boundRefKey, interval) : Filtration.matchNothing(); case NOT_EQUALS: return isAligned ? new NotDimFilter(Bounds.interval(boundRefKey, interval)) : Filtration.matchEverything(); case GREATER_THAN: return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(interval.getEndMillis())); case GREATER_THAN_OR_EQUAL: return isAligned ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(interval.getStartMillis())) : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(interval.getEndMillis())); case LESS_THAN: return isAligned ? Bounds.lessThan(boundRefKey, String.valueOf(interval.getStartMillis())) : Bounds.lessThan(boundRefKey, String.valueOf(interval.getEndMillis())); case LESS_THAN_OR_EQUAL: return Bounds.lessThan(boundRefKey, String.valueOf(interval.getEndMillis())); default: throw new IllegalStateException("WTF?! Shouldn't have got here..."); } }
From source file:org.apache.druid.sql.calcite.filtration.Bounds.java
License:Apache License
public static BoundDimFilter interval(final BoundRefKey boundRefKey, final Interval interval) { if (!boundRefKey.getComparator().equals(StringComparators.NUMERIC)) { // Interval comparison only works with NUMERIC comparator. throw new ISE("Comparator must be NUMERIC but was[%s]", boundRefKey.getComparator()); }//from w w w . j av a2 s . com return new BoundDimFilter(boundRefKey.getDimension(), String.valueOf(interval.getStartMillis()), String.valueOf(interval.getEndMillis()), false, true, null, boundRefKey.getExtractionFn(), boundRefKey.getComparator(), null); }
From source file:org.apache.druid.timeline.SegmentId.java
License:Apache License
private SegmentId(String dataSource, Interval interval, String version, int partitionNum) { this.dataSource = STRING_INTERNER.intern(Objects.requireNonNull(dataSource)); this.intervalStartMillis = interval.getStartMillis(); this.intervalEndMillis = interval.getEndMillis(); this.intervalChronology = interval.getChronology(); // Versions are timestamp-based Strings, interning of them doesn't make sense. If this is not the case, interning // could be conditionally allowed via a system property. this.version = Objects.requireNonNull(version); this.partitionNum = partitionNum; this.hashCode = computeHashCode(); }