Example usage for org.joda.time Interval Interval

Introduction

In this page you can find the example usage for org.joda.time Interval Interval.

Prototype

public Interval(Object interval, Chronology chronology) 

Source Link

Document

Constructs a time interval by converting or copying from another object, overriding the chronology.

Usage

From source file:com.metamx.common.Granularity.java

License:Apache License

public Iterable<Interval> getReverseIterable(final DateTime start, final DateTime end) {
    return getReverseIterable(new Interval(start, end));
}

---

From source file:com.metamx.common.JodaUtils.java

License:Apache License

public static ArrayList<Interval> condenseIntervals(Iterable<Interval> intervals) {
    ArrayList<Interval> retVal = Lists.newArrayList();

    TreeSet<Interval> sortedIntervals = Sets.newTreeSet(Comparators.intervalsByStartThenEnd());
    for (Interval interval : intervals) {
        sortedIntervals.add(interval);//from   ww  w.ja  va  2  s  . c om
    }

    if (sortedIntervals.isEmpty()) {
        return Lists.newArrayList();
    }

    Iterator<Interval> intervalsIter = sortedIntervals.iterator();
    Interval currInterval = intervalsIter.next();
    while (intervalsIter.hasNext()) {
        Interval next = intervalsIter.next();

        if (currInterval.overlaps(next) || currInterval.abuts(next)) {
            currInterval = new Interval(currInterval.getStart(), next.getEnd());
        } else {
            retVal.add(currInterval);
            currInterval = next;
        }
    }
    retVal.add(currInterval);

    return retVal;
}

---

From source file:com.metamx.common.JodaUtils.java

License:Apache License

public static Interval umbrellaInterval(Iterable<Interval> intervals) {
    ArrayList<DateTime> startDates = Lists.newArrayList();
    ArrayList<DateTime> endDates = Lists.newArrayList();

    for (Interval interval : intervals) {
        startDates.add(interval.getStart());
        endDates.add(interval.getEnd());
    }/*  w w w  .j  av a 2  s  .  co m*/

    DateTime minStart = minDateTime(startDates.toArray(new DateTime[] {}));
    DateTime maxEnd = maxDateTime(endDates.toArray(new DateTime[] {}));

    if (minStart == null || maxEnd == null) {
        throw new IllegalArgumentException("Empty list of intervals");
    }
    return new Interval(minStart, maxEnd);
}

---

From source file:com.metamx.druid.db.DatabaseSegmentManager.java

License:Open Source License

public boolean enableDatasource(final String ds) {
    try {//from ww  w .  j a va 2  s . c o  m
        VersionedIntervalTimeline<String, DataSegment> segmentTimeline = dbi
                .withHandle(new HandleCallback<VersionedIntervalTimeline<String, DataSegment>>() {
                    @Override
                    public VersionedIntervalTimeline<String, DataSegment> withHandle(Handle handle)
                            throws Exception {
                        return handle
                                .createQuery(
                                        String.format("SELECT payload FROM %s WHERE dataSource = :dataSource",
                                                config.getSegmentTable()))
                                .bind("dataSource", ds)
                                .fold(new VersionedIntervalTimeline<String, DataSegment>(Ordering.natural()),
                                        new Folder3<VersionedIntervalTimeline<String, DataSegment>, Map<String, Object>>() {
                                            @Override
                                            public VersionedIntervalTimeline<String, DataSegment> fold(
                                                    VersionedIntervalTimeline<String, DataSegment> timeline,
                                                    Map<String, Object> stringObjectMap,
                                                    FoldController foldController,
                                                    StatementContext statementContext) throws SQLException {
                                                try {
                                                    DataSegment segment = jsonMapper.readValue(
                                                            (String) stringObjectMap.get("payload"),
                                                            DataSegment.class);

                                                    timeline.add(segment.getInterval(), segment.getVersion(),
                                                            segment.getShardSpec().createChunk(segment));

                                                    return timeline;
                                                } catch (Exception e) {
                                                    throw new SQLException(e.toString());
                                                }
                                            }
                                        });
                    }
                });

        final List<DataSegment> segments = Lists.transform(
                segmentTimeline.lookup(new Interval(new DateTime(0), new DateTime("3000-01-01"))),
                new Function<TimelineObjectHolder<String, DataSegment>, DataSegment>() {
                    @Override
                    public DataSegment apply(@Nullable TimelineObjectHolder<String, DataSegment> input) {
                        return input.getObject().getChunk(0).getObject();
                    }
                });

        if (segments.isEmpty()) {
            log.warn("No segments found in the database!");
            return false;
        }

        dbi.withHandle(new HandleCallback<Void>() {
            @Override
            public Void withHandle(Handle handle) throws Exception {
                Batch batch = handle.createBatch();

                for (DataSegment segment : segments) {
                    batch.add(String.format("UPDATE %s SET used=1 WHERE id = '%s'", config.getSegmentTable(),
                            segment.getIdentifier()));
                }
                batch.execute();

                return null;
            }
        });
    } catch (Exception e) {
        log.error(e, "Exception enabling datasource %s", ds);
        return false;
    }

    return true;
}

---

From source file:com.metamx.druid.http.ClientInfoResource.java

License:Open Source License

@GET
@Path("/{dataSourceName}/dimensions")
@Produces("application/json")
public Iterable<String> getDatasourceDimensions(@PathParam("dataSourceName") String dataSourceName,
        @QueryParam("interval") String interval) {
    DruidDataSource dataSource = updateDataSources().get(dataSourceName);

    Set<String> retVal = Sets.newHashSet();

    Interval dimInterval;//from ww  w. j  av a2 s.c  o  m
    if (interval == null || interval.isEmpty()) {
        Iterator<DataSegment> iter = Lists.reverse(Lists.newArrayList(dataSource.getSegments())).iterator();
        DataSegment segment = iter.next();
        retVal.addAll(segment.getDimensions());

        dimInterval = new Interval(segment.getInterval().getEnd().minus(SEGMENT_HISTORY_MILLIS),
                segment.getInterval().getEnd());

        while (iter.hasNext() && dimInterval.contains(segment.getInterval())) {
            retVal.addAll(segment.getDimensions());
            segment = iter.next();
        }
    } else {
        try {
            dimInterval = new Interval(interval);
        } catch (Exception e) {
            throw new IAE("Interval is not in a parseable format!");
        }

        Iterator<DataSegment> iter = dataSource.getSegments().iterator();

        while (iter.hasNext()) {
            DataSegment segment = iter.next();
            if (dimInterval.contains(segment.getInterval())) {
                retVal.addAll(segment.getDimensions());
            }
        }
    }

    return retVal;
}

---

From source file:com.metamx.druid.http.ClientInfoResource.java

License:Open Source License

@GET
@Path("/{dataSourceName}/metrics")
@Produces("application/json")
public Iterable<String> getDatasourceMetrics(@PathParam("dataSourceName") String dataSourceName,
        @QueryParam("interval") String interval) {
    DruidDataSource dataSource = updateDataSources().get(dataSourceName);

    Set<String> retVal = Sets.newHashSet();

    Interval dimInterval;/*from w w w  . j av  a 2 s . com*/
    if (interval == null || interval.isEmpty()) {
        Iterator<DataSegment> iter = Lists.reverse(Lists.newArrayList(dataSource.getSegments())).iterator();
        DataSegment segment = iter.next();
        retVal.addAll(segment.getMetrics());

        dimInterval = new Interval(segment.getInterval().getEnd().minus(SEGMENT_HISTORY_MILLIS),
                segment.getInterval().getEnd());

        while (iter.hasNext() && dimInterval.contains(segment.getInterval())) {
            retVal.addAll(segment.getMetrics());
            segment = iter.next();
        }
    } else {
        try {
            dimInterval = new Interval(interval);
        } catch (Exception e) {
            throw new IAE("Interval is not in a parseable format!");
        }

        Iterator<DataSegment> iter = dataSource.getSegments().iterator();

        while (iter.hasNext()) {
            DataSegment segment = iter.next();
            if (dimInterval.contains(segment.getInterval())) {
                retVal.addAll(segment.getMetrics());
            }
        }
    }

    return retVal;
}

---

From source file:com.metamx.druid.index.v1.IncrementalIndex.java

License:Open Source License

public Interval getInterval() {
    return new Interval(minTimestamp, isEmpty() ? minTimestamp : gran.next(getMaxTimeMillis()));
}

---

From source file:com.metamx.druid.index.v1.IndexMerger.java

License:Open Source License

private static File makeIndexFiles(final List<IndexableAdapter> indexes, final File outDir,
        final ProgressIndicator progress, final List<String> mergedDimensions, final List<String> mergedMetrics,
        final Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>> rowMergerFn) throws IOException {
    Map<String, String> metricTypes = Maps.newTreeMap(Ordering.<String>natural().nullsFirst());
    for (IndexableAdapter adapter : indexes) {
        for (String metric : adapter.getAvailableMetrics()) {
            metricTypes.put(metric, adapter.getMetricType(metric));
        }//w  ww  . ja v  a 2 s  .  c o  m
    }
    final Interval dataInterval;
    File v8OutDir = new File(outDir, "v8-tmp");
    v8OutDir.mkdirs();

    /*************  Main index.drd file **************/
    progress.progress();
    long startTime = System.currentTimeMillis();
    File indexFile = new File(v8OutDir, "index.drd");

    FileOutputStream fileOutputStream = null;
    FileChannel channel = null;
    try {
        fileOutputStream = new FileOutputStream(indexFile);
        channel = fileOutputStream.getChannel();
        channel.write(ByteBuffer.wrap(new byte[] { IndexIO.V8_VERSION }));

        GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.stringStrategy).writeToChannel(channel);
        GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.stringStrategy).writeToChannel(channel);

        DateTime minTime = new DateTime(Long.MAX_VALUE);
        DateTime maxTime = new DateTime(0l);

        for (IndexableAdapter index : indexes) {
            minTime = JodaUtils.minDateTime(minTime, index.getDataInterval().getStart());
            maxTime = JodaUtils.maxDateTime(maxTime, index.getDataInterval().getEnd());
        }

        dataInterval = new Interval(minTime, maxTime);
        serializerUtils.writeString(channel, String.format("%s/%s", minTime, maxTime));
    } finally {
        Closeables.closeQuietly(channel);
        channel = null;
        Closeables.closeQuietly(fileOutputStream);
        fileOutputStream = null;
    }
    IndexIO.checkFileSize(indexFile);
    log.info("outDir[%s] completed index.drd in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime);

    /************* Setup Dim Conversions **************/
    progress.progress();
    startTime = System.currentTimeMillis();

    IOPeon ioPeon = new TmpFileIOPeon();
    ArrayList<FileOutputSupplier> dimOuts = Lists.newArrayListWithCapacity(mergedDimensions.size());
    Map<String, Integer> dimensionCardinalities = Maps.newHashMap();
    ArrayList<Map<String, IntBuffer>> dimConversions = Lists.newArrayListWithCapacity(indexes.size());

    for (IndexableAdapter index : indexes) {
        dimConversions.add(Maps.<String, IntBuffer>newHashMap());
    }

    for (String dimension : mergedDimensions) {
        final FlattenedArrayWriter<String> writer = new FlattenedArrayWriter<String>(ioPeon, dimension,
                GenericIndexed.stringStrategy);
        writer.open();

        List<Indexed<String>> dimValueLookups = Lists.newArrayListWithCapacity(indexes.size());
        DimValueConverter[] converters = new DimValueConverter[indexes.size()];
        for (int i = 0; i < indexes.size(); i++) {
            Indexed<String> dimValues = indexes.get(i).getDimValueLookup(dimension);
            if (dimValues != null) {
                dimValueLookups.add(dimValues);
                converters[i] = new DimValueConverter(dimValues);
            }
        }

        Iterable<String> dimensionValues = CombiningIterable.createSplatted(
                Iterables.transform(dimValueLookups, new Function<Indexed<String>, Iterable<String>>() {
                    @Override
                    public Iterable<String> apply(@Nullable Indexed<String> indexed) {
                        return Iterables.transform(indexed, new Function<String, String>() {
                            @Override
                            public String apply(@Nullable String input) {
                                return (input == null) ? "" : input;
                            }
                        });
                    }
                }), Ordering.<String>natural().nullsFirst());

        int count = 0;
        for (String value : dimensionValues) {
            value = value == null ? "" : value;
            writer.write(value);

            for (int i = 0; i < indexes.size(); i++) {
                DimValueConverter converter = converters[i];
                if (converter != null) {
                    converter.convert(value, count);
                }
            }

            ++count;
        }
        dimensionCardinalities.put(dimension, count);

        FileOutputSupplier dimOut = new FileOutputSupplier(IndexIO.makeDimFile(v8OutDir, dimension), true);
        dimOuts.add(dimOut);

        writer.close();
        serializerUtils.writeString(dimOut, dimension);
        ByteStreams.copy(writer.combineStreams(), dimOut);
        for (int i = 0; i < indexes.size(); ++i) {
            DimValueConverter converter = converters[i];
            if (converter != null) {
                dimConversions.get(i).put(dimension, converters[i].getConversionBuffer());
            }
        }

        ioPeon.cleanup();
    }
    log.info("outDir[%s] completed dim conversions in %,d millis.", v8OutDir,
            System.currentTimeMillis() - startTime);

    /************* Walk through data sets and merge them *************/
    progress.progress();
    startTime = System.currentTimeMillis();

    ArrayList<Iterable<Rowboat>> boats = Lists.newArrayListWithCapacity(indexes.size());

    for (int i = 0; i < indexes.size(); ++i) {
        final IndexableAdapter adapter = indexes.get(i);

        final int[] dimLookup = new int[mergedDimensions.size()];
        int count = 0;
        for (String dim : adapter.getAvailableDimensions()) {
            dimLookup[count] = mergedDimensions.indexOf(dim.toLowerCase());
            count++;
        }

        final int[] metricLookup = new int[mergedMetrics.size()];
        count = 0;
        for (String metric : adapter.getAvailableMetrics()) {
            metricLookup[count] = mergedMetrics.indexOf(metric);
            count++;
        }

        boats.add(new MMappedIndexRowIterable(
                Iterables.transform(indexes.get(i).getRows(), new Function<Rowboat, Rowboat>() {
                    @Override
                    public Rowboat apply(@Nullable Rowboat input) {
                        int[][] newDims = new int[mergedDimensions.size()][];
                        int j = 0;
                        for (int[] dim : input.getDims()) {
                            newDims[dimLookup[j]] = dim;
                            j++;
                        }

                        Object[] newMetrics = new Object[mergedMetrics.size()];
                        j = 0;
                        for (Object met : input.getMetrics()) {
                            newMetrics[metricLookup[j]] = met;
                            j++;
                        }

                        return new Rowboat(input.getTimestamp(), newDims, newMetrics, input.getRowNum());
                    }
                }), mergedDimensions, dimConversions.get(i), i));
    }

    Iterable<Rowboat> theRows = rowMergerFn.apply(boats);

    CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(ioPeon,
            "little_end_time", IndexIO.BYTE_ORDER);

    timeWriter.open();

    ArrayList<VSizeIndexedWriter> forwardDimWriters = Lists.newArrayListWithCapacity(mergedDimensions.size());
    for (String dimension : mergedDimensions) {
        VSizeIndexedWriter writer = new VSizeIndexedWriter(ioPeon, dimension,
                dimensionCardinalities.get(dimension));
        writer.open();
        forwardDimWriters.add(writer);
    }

    ArrayList<MetricColumnSerializer> metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size());
    for (Map.Entry<String, String> entry : metricTypes.entrySet()) {
        String metric = entry.getKey();
        String typeName = entry.getValue();
        if ("float".equals(typeName)) {
            metWriters.add(new FloatMetricColumnSerializer(metric, v8OutDir, ioPeon));
        } else {
            ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);

            if (serde == null) {
                throw new ISE("Unknown type[%s]", typeName);
            }

            metWriters.add(new ComplexMetricColumnSerializer(metric, v8OutDir, ioPeon, serde));
        }
    }
    for (MetricColumnSerializer metWriter : metWriters) {
        metWriter.open();
    }

    int rowCount = 0;
    long time = System.currentTimeMillis();
    List<IntBuffer> rowNumConversions = Lists.newArrayListWithCapacity(indexes.size());
    for (IndexableAdapter index : indexes) {
        int[] arr = new int[index.getNumRows()];
        Arrays.fill(arr, INVALID_ROW);
        rowNumConversions.add(IntBuffer.wrap(arr));
    }

    for (Rowboat theRow : theRows) {
        progress.progress();
        timeWriter.add(theRow.getTimestamp());

        final Object[] metrics = theRow.getMetrics();
        for (int i = 0; i < metrics.length; ++i) {
            metWriters.get(i).serialize(metrics[i]);
        }

        int[][] dims = theRow.getDims();
        for (int i = 0; i < dims.length; ++i) {
            List<Integer> listToWrite = (i >= dims.length || dims[i] == null) ? null : Ints.asList(dims[i]);
            forwardDimWriters.get(i).write(listToWrite);
        }

        for (Map.Entry<Integer, TreeSet<Integer>> comprisedRow : theRow.getComprisedRows().entrySet()) {
            final IntBuffer conversionBuffer = rowNumConversions.get(comprisedRow.getKey());

            for (Integer rowNum : comprisedRow.getValue()) {
                while (conversionBuffer.position() < rowNum) {
                    conversionBuffer.put(INVALID_ROW);
                }
                conversionBuffer.put(rowCount);
            }
        }

        if ((++rowCount % 500000) == 0) {
            log.info("outDir[%s] walked 500,000/%,d rows in %,d millis.", v8OutDir, rowCount,
                    System.currentTimeMillis() - time);
            time = System.currentTimeMillis();
        }
    }

    for (IntBuffer rowNumConversion : rowNumConversions) {
        rowNumConversion.rewind();
    }

    final File timeFile = IndexIO.makeTimeFile(v8OutDir, IndexIO.BYTE_ORDER);
    timeFile.delete();
    OutputSupplier<FileOutputStream> out = Files.newOutputStreamSupplier(timeFile, true);
    timeWriter.closeAndConsolidate(out);
    IndexIO.checkFileSize(timeFile);

    for (int i = 0; i < mergedDimensions.size(); ++i) {
        forwardDimWriters.get(i).close();
        ByteStreams.copy(forwardDimWriters.get(i).combineStreams(), dimOuts.get(i));
    }

    for (MetricColumnSerializer metWriter : metWriters) {
        metWriter.close();
    }

    ioPeon.cleanup();
    log.info("outDir[%s] completed walk through of %,d rows in %,d millis.", v8OutDir, rowCount,
            System.currentTimeMillis() - startTime);

    /************ Create Inverted Indexes *************/
    startTime = System.currentTimeMillis();

    final File invertedFile = new File(v8OutDir, "inverted.drd");
    Files.touch(invertedFile);
    out = Files.newOutputStreamSupplier(invertedFile, true);
    for (int i = 0; i < mergedDimensions.size(); ++i) {
        long dimStartTime = System.currentTimeMillis();
        String dimension = mergedDimensions.get(i);

        File dimOutFile = dimOuts.get(i).getFile();
        final MappedByteBuffer dimValsMapped = Files.map(dimOutFile);

        if (!dimension.equals(serializerUtils.readString(dimValsMapped))) {
            throw new ISE("dimensions[%s] didn't equate!?  This is a major WTF moment.", dimension);
        }
        Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.stringStrategy);
        log.info("Starting dimension[%s] with cardinality[%,d]", dimension, dimVals.size());

        FlattenedArrayWriter<ImmutableConciseSet> writer = new FlattenedArrayWriter<ImmutableConciseSet>(ioPeon,
                dimension, ConciseCompressedIndexedInts.objectStrategy);
        writer.open();

        for (String dimVal : IndexedIterable.create(dimVals)) {
            progress.progress();
            List<Iterable<Integer>> convertedInverteds = Lists.newArrayListWithCapacity(indexes.size());
            for (int j = 0; j < indexes.size(); ++j) {
                convertedInverteds.add(new ConvertingIndexedInts(indexes.get(j).getInverteds(dimension, dimVal),
                        rowNumConversions.get(j)));
            }

            ConciseSet bitset = new ConciseSet();
            for (Integer row : CombiningIterable.createSplatted(convertedInverteds,
                    Ordering.<Integer>natural().nullsFirst())) {
                if (row != INVALID_ROW) {
                    bitset.add(row);
                }
            }

            writer.write(ImmutableConciseSet.newImmutableFromMutable(bitset));
        }
        writer.close();

        serializerUtils.writeString(out, dimension);
        ByteStreams.copy(writer.combineStreams(), out);
        ioPeon.cleanup();

        log.info("Completed dimension[%s] in %,d millis.", dimension,
                System.currentTimeMillis() - dimStartTime);
    }
    log.info("outDir[%s] completed inverted.drd in %,d millis.", v8OutDir,
            System.currentTimeMillis() - startTime);

    final ArrayList<String> expectedFiles = Lists.newArrayList(Iterables.concat(
            Arrays.asList("index.drd", "inverted.drd", String.format("time_%s.drd", IndexIO.BYTE_ORDER)),
            Iterables.transform(mergedDimensions, GuavaUtils.formatFunction("dim_%s.drd")),
            Iterables.transform(mergedMetrics,
                    GuavaUtils.formatFunction(String.format("met_%%s_%s.drd", IndexIO.BYTE_ORDER)))));

    Map<String, File> files = Maps.newLinkedHashMap();
    for (String fileName : expectedFiles) {
        files.put(fileName, new File(v8OutDir, fileName));
    }

    File smooshDir = new File(v8OutDir, "smoosher");
    smooshDir.mkdir();

    for (Map.Entry<String, File> entry : Smoosh.smoosh(v8OutDir, smooshDir, files).entrySet()) {
        entry.getValue().delete();
    }

    for (File file : smooshDir.listFiles()) {
        Files.move(file, new File(v8OutDir, file.getName()));
    }

    if (!smooshDir.delete()) {
        log.info("Unable to delete temporary dir[%s], contains[%s]", smooshDir,
                Arrays.asList(smooshDir.listFiles()));
        throw new IOException(String.format("Unable to delete temporary dir[%s]", smooshDir));
    }

    createIndexDrdFile(IndexIO.V8_VERSION, v8OutDir,
            GenericIndexed.fromIterable(mergedDimensions, GenericIndexed.stringStrategy),
            GenericIndexed.fromIterable(mergedMetrics, GenericIndexed.stringStrategy), dataInterval);

    IndexIO.DefaultIndexIOHandler.convertV8toV9(v8OutDir, outDir);
    FileUtils.deleteDirectory(v8OutDir);

    return outDir;
}

---

From source file:com.metamx.druid.indexer.granularity.ArbitraryGranularitySpec.java

License:Open Source License

@Override
public Optional<Interval> bucketInterval(DateTime dt) {
    // First interval with start time  dt
    final Interval interval = intervals.floor(new Interval(dt, new DateTime(Long.MAX_VALUE)));

    if (interval != null && interval.contains(dt)) {
        return Optional.of(interval);
    } else {/*from  ww  w.  jav  a2  s .  c o m*/
        return Optional.absent();
    }
}

---

From source file:com.metamx.druid.indexer.path.GranularUnprocessedPathSpec.java

License:Open Source License

@Override
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException {
    // This PathSpec breaks so many abstractions that we might as break some more
    Preconditions.checkState(config.getGranularitySpec() instanceof UniformGranularitySpec,
            String.format("Cannot use %s without %s", GranularUnprocessedPathSpec.class.getSimpleName(),
                    UniformGranularitySpec.class.getSimpleName()));

    final Path betaInput = new Path(getInputPath());
    final FileSystem fs = betaInput.getFileSystem(job.getConfiguration());
    final Granularity segmentGranularity = ((UniformGranularitySpec) config.getGranularitySpec())
            .getGranularity();/*from  www. jav  a  2  s .c o  m*/

    Map<DateTime, Long> inputModifiedTimes = new TreeMap<DateTime, Long>(
            Comparators.inverse(Comparators.<Comparable>comparable()));

    for (FileStatus status : FSSpideringIterator.spiderIterable(fs, betaInput)) {
        final DateTime key = segmentGranularity.toDate(status.getPath().toString());
        final Long currVal = inputModifiedTimes.get(key);
        final long mTime = status.getModificationTime();

        inputModifiedTimes.put(key, currVal == null ? mTime : Math.max(currVal, mTime));
    }

    Set<Interval> bucketsToRun = Sets.newTreeSet(Comparators.intervals());
    for (Map.Entry<DateTime, Long> entry : inputModifiedTimes.entrySet()) {
        DateTime timeBucket = entry.getKey();
        long mTime = entry.getValue();

        String bucketOutput = String.format("%s/%s", config.getSegmentOutputDir(),
                segmentGranularity.toPath(timeBucket));
        for (FileStatus fileStatus : FSSpideringIterator.spiderIterable(fs, new Path(bucketOutput))) {
            if (fileStatus.getModificationTime() > mTime) {
                bucketsToRun.add(new Interval(timeBucket, segmentGranularity.increment(timeBucket)));
                break;
            }
        }

        if (bucketsToRun.size() >= maxBuckets) {
            break;
        }
    }

    config.setGranularitySpec(new UniformGranularitySpec(segmentGranularity, Lists.newArrayList(bucketsToRun)));

    return super.addInputPaths(config, job);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• »