List of usage examples for org.joda.time Interval getEndMillis
public long getEndMillis()
From source file:org.apache.druid.indexer.path.GranularityPathSpec.java
License:Apache License
private Interval trim(Interval inputInterval, Interval interval) { long start = interval.getStartMillis(); long end = interval.getEndMillis(); boolean makeNew = false; if (start < inputInterval.getStartMillis()) { start = inputInterval.getStartMillis(); makeNew = true;/* w w w . j ava 2 s . com*/ } if (end > inputInterval.getEndMillis()) { end = inputInterval.getEndMillis(); makeNew = true; } return makeNew ? new Interval(start, end, interval.getChronology()) : interval; }
From source file:org.apache.druid.java.util.common.granularity.ArbitraryGranularity.java
License:Apache License
@JsonCreator public ArbitraryGranularity(@JsonProperty("intervals") List<Interval> inputIntervals, @JsonProperty("timezone") DateTimeZone timezone) { this.intervals = new TreeSet<>(Comparators.intervalsByStartThenEnd()); this.timezone = timezone == null ? DateTimeZone.UTC : timezone; if (inputIntervals == null) { inputIntervals = new ArrayList<>(); }//from w ww . ja va 2 s .c om Preconditions.checkArgument(inputIntervals.size() > 0, "at least one interval should be specified"); // Insert all intervals for (final Interval inputInterval : inputIntervals) { Interval adjustedInterval = inputInterval; if (timezone != null) { adjustedInterval = new Interval(inputInterval.getStartMillis(), inputInterval.getEndMillis(), timezone); } intervals.add(adjustedInterval); } // Ensure intervals are non-overlapping (but they may abut each other) final PeekingIterator<Interval> intervalIterator = Iterators.peekingIterator(intervals.iterator()); while (intervalIterator.hasNext()) { final Interval currentInterval = intervalIterator.next(); if (intervalIterator.hasNext()) { final Interval nextInterval = intervalIterator.peek(); if (currentInterval.overlaps(nextInterval)) { throw new IAE("Overlapping granularity intervals: %s, %s", currentInterval, nextInterval); } } } }
From source file:org.apache.druid.java.util.common.granularity.ArbitraryGranularity.java
License:Apache License
@Override public Iterable<Interval> getIterable(Interval input) { long start = input.getStartMillis(); long end = input.getEndMillis(); // Return an empty iterable if the requested time interval does not // overlap any of the arbitrary intervals specified if (end < intervals.first().getStartMillis() || start > intervals.last().getEndMillis()) { return ImmutableList.of(); }/*from w w w. j a v a2 s . c o m*/ return new Iterable<Interval>() { @Override public Iterator<Interval> iterator() { // Skip over the intervals that are known to be invalid // because they end before the requested start timestamp final PeekingIterator<Interval> intervalIterator = Iterators.peekingIterator(intervals.iterator()); while (intervalIterator.hasNext() && intervalIterator.peek().getEndMillis() <= start) { intervalIterator.next(); } return new Iterator<Interval>() { @Override public boolean hasNext() { return intervalIterator.hasNext() && intervalIterator.peek().getStartMillis() < end; } @Override public Interval next() { return intervalIterator.next(); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }; }
From source file:org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator.java
License:Apache License
@Nullable private SegmentIdWithShardSpec allocatePendingSegment(final Handle handle, final String dataSource, final String sequenceName, final Interval interval, final ShardSpecFactory shardSpecFactory, final String maxVersion) throws IOException { final CheckExistingSegmentIdResult result = checkAndGetExistingSegmentId( handle.createQuery(StringUtils.format( "SELECT payload FROM %s WHERE " + "dataSource = :dataSource AND " + "sequence_name = :sequence_name AND " + "start = :start AND " + "%2$send%2$s = :end", dbTables.getPendingSegmentsTable(), connector.getQuoteString())), interval, sequenceName, null, Pair.of("dataSource", dataSource), Pair.of("sequence_name", sequenceName), Pair.of("start", interval.getStart().toString()), Pair.of("end", interval.getEnd().toString())); if (result.found) { // The found existing segment identifier can be null if its interval doesn't match with the given interval return result.segmentIdentifier; }/*from ww w . jav a 2s .co m*/ final SegmentIdWithShardSpec newIdentifier = createNewSegment(handle, dataSource, interval, shardSpecFactory, maxVersion); if (newIdentifier == null) { return null; } // SELECT -> INSERT can fail due to races; callers must be prepared to retry. // Avoiding ON DUPLICATE KEY since it's not portable. // Avoiding try/catch since it may cause inadvertent transaction-splitting. // UNIQUE key for the row, ensuring we don't have more than one segment per sequence per interval. // Using a single column instead of (sequence_name, sequence_prev_id) as some MySQL storage engines // have difficulty with large unique keys (see https://github.com/apache/incubator-druid/issues/2319) final String sequenceNamePrevIdSha1 = BaseEncoding.base16() .encode(Hashing.sha1().newHasher().putBytes(StringUtils.toUtf8(sequenceName)).putByte((byte) 0xff) .putLong(interval.getStartMillis()).putLong(interval.getEndMillis()).hash().asBytes()); // always insert empty previous sequence id insertToMetastore(handle, newIdentifier, dataSource, interval, "", sequenceName, sequenceNamePrevIdSha1); log.info("Allocated pending segment [%s] for sequence[%s] in DB", newIdentifier, sequenceName); return newIdentifier; }
From source file:org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator.java
License:Apache License
private CheckExistingSegmentIdResult checkAndGetExistingSegmentId(final Query<Map<String, Object>> query, final Interval interval, final String sequenceName, final @Nullable String previousSegmentId, final Pair<String, String>... queryVars) throws IOException { Query<Map<String, Object>> boundQuery = query; for (Pair<String, String> var : queryVars) { boundQuery = boundQuery.bind(var.lhs, var.rhs); }/*w w w.j a v a 2 s. co m*/ final List<byte[]> existingBytes = boundQuery.map(ByteArrayMapper.FIRST).list(); if (!existingBytes.isEmpty()) { final SegmentIdWithShardSpec existingIdentifier = jsonMapper .readValue(Iterables.getOnlyElement(existingBytes), SegmentIdWithShardSpec.class); if (existingIdentifier.getInterval().getStartMillis() == interval.getStartMillis() && existingIdentifier.getInterval().getEndMillis() == interval.getEndMillis()) { if (previousSegmentId == null) { log.info("Found existing pending segment [%s] for sequence[%s] in DB", existingIdentifier, sequenceName); } else { log.info("Found existing pending segment [%s] for sequence[%s] (previous = [%s]) in DB", existingIdentifier, sequenceName, previousSegmentId); } return new CheckExistingSegmentIdResult(true, existingIdentifier); } else { if (previousSegmentId == null) { log.warn( "Cannot use existing pending segment [%s] for sequence[%s] in DB, " + "does not match requested interval[%s]", existingIdentifier, sequenceName, interval); } else { log.warn( "Cannot use existing pending segment [%s] for sequence[%s] (previous = [%s]) in DB, " + "does not match requested interval[%s]", existingIdentifier, sequenceName, previousSegmentId, interval); } return new CheckExistingSegmentIdResult(true, null); } } return new CheckExistingSegmentIdResult(false, null); }
From source file:org.apache.druid.query.filter.IntervalDimFilter.java
License:Apache License
private List<Pair<Long, Long>> makeIntervalLongs() { List<Pair<Long, Long>> intervalLongs = new ArrayList<>(); for (Interval interval : intervals) { intervalLongs.add(new Pair<>(interval.getStartMillis(), interval.getEndMillis())); }//from www . j ava 2 s .co m return intervalLongs; }
From source file:org.apache.druid.query.IntervalChunkingQueryRunner.java
License:Apache License
private static Iterable<Interval> splitInterval(Interval interval, Period period) { if (interval.getEndMillis() == interval.getStartMillis()) { return Collections.singletonList(interval); }// www. j a va2 s. com List<Interval> intervals = new ArrayList<>(); Iterator<Interval> timestamps = new PeriodGranularity(period, null, null).getIterable(interval).iterator(); DateTime start = DateTimes.max(timestamps.next().getStart(), interval.getStart()); while (timestamps.hasNext()) { DateTime end = timestamps.next().getStart(); intervals.add(new Interval(start, end)); start = end; } if (start.compareTo(interval.getEnd()) < 0) { intervals.add(new Interval(start, interval.getEnd())); } return intervals; }
From source file:org.apache.druid.query.materializedview.MaterializedViewUtils.java
License:Apache License
/** * calculate the intervals which are covered by interval2, but not covered by interval1. * result intervals = interval2 - interval1 interval2 * e.g. /*from w w w . j av a2 s. c o m*/ * a list of interval2: ["2018-04-01T00:00:00.000Z/2018-04-02T00:00:00.000Z", * "2018-04-03T00:00:00.000Z/2018-04-10T00:00:00.000Z"] * a list of interval1: ["2018-04-04T00:00:00.000Z/2018-04-06T00:00:00.000Z"] * the result list of intervals: ["2018-04-01T00:00:00.000Z/2018-04-02T00:00:00.000Z", * "2018-04-03T00:00:00.000Z/2018-04-04T00:00:00.000Z", * "2018-04-06T00:00:00.000Z/2018-04-10T00:00:00.000Z"] * If interval2 is empty, then return an empty list of interval. * @param interval2 list of intervals * @param interval1 list of intervals * @return list of intervals are covered by interval2, but not covered by interval1. */ public static List<Interval> minus(List<Interval> interval2, List<Interval> interval1) { if (interval1.isEmpty() || interval2.isEmpty()) { return interval1; } Iterator<Interval> it1 = JodaUtils.condenseIntervals(interval1).iterator(); Iterator<Interval> it2 = JodaUtils.condenseIntervals(interval2).iterator(); List<Interval> remaining = new ArrayList<>(); Interval currInterval1 = it1.next(); Interval currInterval2 = it2.next(); long start1 = currInterval1.getStartMillis(); long end1 = currInterval1.getEndMillis(); long start2 = currInterval2.getStartMillis(); long end2 = currInterval2.getEndMillis(); while (true) { if (start2 < start1 && end2 <= start1) { remaining.add(Intervals.utc(start2, end2)); if (it2.hasNext()) { currInterval2 = it2.next(); start2 = currInterval2.getStartMillis(); end2 = currInterval2.getEndMillis(); } else { break; } } if (start2 < start1 && end2 > start1 && end2 < end1) { remaining.add(Intervals.utc(start2, start1)); start1 = end2; if (it2.hasNext()) { currInterval2 = it2.next(); start2 = currInterval2.getStartMillis(); end2 = currInterval2.getEndMillis(); } else { break; } } if (start2 < start1 && end2 == end1) { remaining.add(Intervals.utc(start2, start1)); if (it2.hasNext() && it1.hasNext()) { currInterval2 = it2.next(); start2 = currInterval2.getStartMillis(); end2 = currInterval2.getEndMillis(); currInterval1 = it1.next(); start1 = currInterval1.getStartMillis(); end1 = currInterval1.getEndMillis(); } else { break; } } if (start2 < start1 && end2 > end1) { remaining.add(Intervals.utc(start2, start1)); start2 = end1; if (it1.hasNext()) { currInterval1 = it1.next(); start1 = currInterval1.getStartMillis(); end1 = currInterval1.getEndMillis(); } else { remaining.add(Intervals.utc(end1, end2)); break; } } if (start2 == start1 && end2 >= start1 && end2 < end1) { start1 = end2; if (it2.hasNext()) { currInterval2 = it2.next(); start2 = currInterval2.getStartMillis(); end2 = currInterval2.getEndMillis(); } else { break; } } if (start2 == start1 && end2 > end1) { start2 = end1; if (it1.hasNext()) { currInterval1 = it1.next(); start1 = currInterval1.getStartMillis(); end1 = currInterval1.getEndMillis(); } else { remaining.add(Intervals.utc(end1, end2)); break; } } if (start2 > start1 && start2 < end1 && end2 < end1) { start1 = end2; if (it2.hasNext()) { currInterval2 = it2.next(); start2 = currInterval2.getStartMillis(); end2 = currInterval2.getEndMillis(); } else { break; } } if (start2 > start1 && start2 < end1 && end2 > end1) { start2 = end1; if (it1.hasNext()) { currInterval1 = it1.next(); start1 = currInterval1.getStartMillis(); end1 = currInterval1.getEndMillis(); } else { remaining.add(Intervals.utc(end1, end2)); break; } } if (start2 >= start1 && start2 <= end1 && end2 == end1) { if (it2.hasNext() && it1.hasNext()) { currInterval2 = it2.next(); start2 = currInterval2.getStartMillis(); end2 = currInterval2.getEndMillis(); currInterval1 = it1.next(); start1 = currInterval1.getStartMillis(); end1 = currInterval1.getEndMillis(); } else { break; } } if (start2 >= end1 && end2 > end1) { if (it1.hasNext()) { currInterval1 = it1.next(); start1 = currInterval1.getStartMillis(); end1 = currInterval1.getEndMillis(); } else { remaining.add(Intervals.utc(start2, end2)); break; } } } while (it2.hasNext()) { remaining.add(Intervals.of(it2.next().toString())); } return remaining; }
From source file:org.apache.druid.query.search.UseIndexesStrategy.java
License:Apache License
static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, final Segment segment, final Filter filter, final Interval interval) { final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); final ImmutableBitmap baseFilter; if (filter == null) { baseFilter = null;//from w ww. j a v a2 s .co m } else { final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index); Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter); baseFilter = filter.getBitmapIndex(selector); } final ImmutableBitmap timeFilteredBitmap; if (!interval.contains(segment.getDataInterval())) { final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap(); final ColumnHolder timeColumnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME); try (final NumericColumn timeValues = (NumericColumn) timeColumnHolder.getColumn()) { int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true)); int endIndex = Math.min(timeValues.length() - 1, getStartIndexOfTime(timeValues, interval.getEndMillis(), false)); for (int i = startIndex; i <= endIndex; i++) { timeBitmap.add(i); } final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap); timeFilteredBitmap = (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter); } } else { timeFilteredBitmap = baseFilter; } return timeFilteredBitmap; }
From source file:org.apache.druid.query.select.SelectQueryQueryToolChest.java
License:Apache License
@Override public <T extends LogicalSegment> List<T> filterSegments(SelectQuery query, List<T> segments) { // at the point where this code is called, only one datasource should exist. final String dataSource = Iterables.getOnlyElement(query.getDataSource().getNames()); PagingSpec pagingSpec = query.getPagingSpec(); Map<String, Integer> paging = pagingSpec.getPagingIdentifiers(); if (paging == null || paging.isEmpty()) { return segments; }//w ww . j a v a2 s .com final Granularity granularity = query.getGranularity(); TreeMap<Long, Long> granularThresholds = new TreeMap<>(); // A paged select query using a UnionDataSource will return pagingIdentifiers from segments in more than one // dataSource which confuses subsequent queries and causes a failure. To avoid this, filter only the paging keys // that are applicable to this dataSource so that each dataSource in a union query gets the appropriate keys. paging.keySet().stream().filter(identifier -> SegmentId.tryParse(dataSource, identifier) != null) .map(SegmentId.makeIntervalExtractor(dataSource)) .sorted(query.isDescending() ? Comparators.intervalsByEndThenStart() : Comparators.intervalsByStartThenEnd()) .forEach(interval -> { if (query.isDescending()) { long granularEnd = granularity.bucketStart(interval.getEnd()).getMillis(); Long currentEnd = granularThresholds.get(granularEnd); if (currentEnd == null || interval.getEndMillis() > currentEnd) { granularThresholds.put(granularEnd, interval.getEndMillis()); } } else { long granularStart = granularity.bucketStart(interval.getStart()).getMillis(); Long currentStart = granularThresholds.get(granularStart); if (currentStart == null || interval.getStartMillis() < currentStart) { granularThresholds.put(granularStart, interval.getStartMillis()); } } }); List<T> queryIntervals = Lists.newArrayList(segments); Iterator<T> it = queryIntervals.iterator(); if (query.isDescending()) { while (it.hasNext()) { Interval interval = it.next().getInterval(); Map.Entry<Long, Long> ceiling = granularThresholds .ceilingEntry(granularity.bucketStart(interval.getEnd()).getMillis()); if (ceiling == null || interval.getStartMillis() >= ceiling.getValue()) { it.remove(); } } } else { while (it.hasNext()) { Interval interval = it.next().getInterval(); Map.Entry<Long, Long> floor = granularThresholds .floorEntry(granularity.bucketStart(interval.getStart()).getMillis()); if (floor == null || interval.getEndMillis() <= floor.getValue()) { it.remove(); } } } return queryIntervals; }