List of usage examples for org.apache.commons.lang3.mutable MutableLong MutableLong
public MutableLong()
From source file:nl.opengeogroep.filesetsync.client.FilesetSyncer.java
/** * Removes files which are locally up-to-date from the list of files to * transfer. Updates lastModified date./*w w w. j a v a2s. c om*/ */ private void compareFilesetList() throws IOException { MutableLong hashTime = new MutableLong(); long hashBytes = 0; long startTime = System.currentTimeMillis(); long progressTime = startTime; int processed = 0; int newerLocalFiles = 0; boolean setLastModifiedToServer = "true".equals(fs.getProperty("setLastModifiedToServer")); for (int index = 0; index < fileList.size(); index++) { FileRecord fr = fileList.get(index); if (Shutdown.isHappening()) { return; } File localFile; if (fileList.size() == 1 && fr.getType() == TYPE_FILE) { localFile = new File(fs.getLocal()); } else { localFile = new File(fs.getLocal() + File.separator + fr.getName()); } if (fr.getType() == TYPE_DIRECTORY && localFile.exists()) { if (!localFile.isDirectory()) { log.error("Local file in is the way for remote directory: " + localFile.getCanonicalPath()); } if (fr.getLastModified() != localFile.lastModified()) { log.trace(String.format("later updating last modified for directory %s", localFile.getCanonicalPath())); directoriesLastModifiedTimes.add(Pair.of(localFile, fr.getLastModified())); } fileList.set(index, null); alreadyLocal++; } if (fr.getType() == TYPE_FILE && localFile.exists()) { if (!localFile.isFile()) { log.error("Local non-file is in the way for remote file: " + localFile.getCanonicalPath()); } if (fs.isHash()) { try { String hash = FileRecord.calculateHash(localFile, hashTime); //localFilesByHash.put(hash, localFile.getCanonicalPath()); hashBytes += localFile.length(); if (hash.equals(fr.getHash())) { if (log.isTraceEnabled()) { log.trace("Same hash for " + fr.getName()); } if (fr.getLastModified() > localFile.lastModified()) { if (log.isTraceEnabled()) { log.trace("Same hash, updating last modified for " + fr.getName()); } localFile.setLastModified(fr.getLastModified()); } fileList.set(index, null); alreadyLocal++; } else { if (log.isTraceEnabled()) { log.trace("Hash mismatch for " + fr.getName()); } } } catch (Exception e) { log.error("Error hashing " + localFile.getCanonicalPath() + ": " + ExceptionUtils.getMessage(e)); } } else { if (fr.getLastModified() > localFile.lastModified()) { if (log.isTraceEnabled()) { log.trace("Remote file newer: " + fr.getName()); } } else if (fr.getLastModified() < localFile.lastModified()) { if (setLastModifiedToServer) { localFile.setLastModified(fr.getLastModified()); } else { if (log.isTraceEnabled()) { log.trace(String.format( "Keeping local file last modified at %s, later than remote file at %s: ", dateToString(new Date(localFile.lastModified())), dateToString(new Date(fr.getLastModified())), fr.getName())); } } newerLocalFiles++; fileList.set(index, null); alreadyLocal++; } else { if (log.isTraceEnabled()) { log.trace("Local file unmodified: " + fr.getName()); } fileList.set(index, null); alreadyLocal++; } } } processed++; long time = System.currentTimeMillis(); if (time - progressTime > 30000) { log.info(String.format("Still comparing files, processed %d files", processed)); progressTime = time; } } // TODO: if file in file list already in localFilesByHash OR, remove them // Also remove duplicate hashes in fileList String hashInfo; if (fs.isHash()) { hashInfo = String.format(", hashed hashed %d KB, hash speed %s", hashBytes / 1024, (hashTime.getValue() < 100 ? "n/a" : Math.round(hashBytes / 1024.0 / (hashTime.getValue() / 1000.0)) + " KB/s")); } else { hashInfo = ""; } log.info(String.format("Compared file list to local files in %s, %d files up-to-date%s", DurationFormatUtils.formatDurationWords(System.currentTimeMillis() - startTime, true, false), alreadyLocal, hashInfo)); if (newerLocalFiles != 0) { log.warn(String.format( "Not overwriting %d local files with newer local last modified date compared to files on server", newerLocalFiles)); } }
From source file:org.apache.drill.exec.store.parquet.ParquetGroupScanStatistics.java
public void collect(List<RowGroupInfo> rowGroupInfos, ParquetTableMetadataBase parquetTableMetadata) { resetHolders();//from ww w . j a va 2 s . c o m boolean first = true; for (RowGroupInfo rowGroup : rowGroupInfos) { long rowCount = rowGroup.getRowCount(); for (ColumnMetadata column : rowGroup.getColumns()) { SchemaPath schemaPath = SchemaPath.getCompoundPath(column.getName()); MutableLong emptyCount = new MutableLong(); MutableLong previousCount = columnValueCounts.putIfAbsent(schemaPath, emptyCount); if (previousCount == null) { previousCount = emptyCount; } if (previousCount.longValue() != GroupScan.NO_COLUMN_STATS && column.isNumNullsSet()) { previousCount.add(rowCount - column.getNulls()); } else { previousCount.setValue(GroupScan.NO_COLUMN_STATS); } boolean partitionColumn = checkForPartitionColumn(column, first, rowCount, parquetTableMetadata); if (partitionColumn) { Map<SchemaPath, Object> map = partitionValueMap.computeIfAbsent(rowGroup.getPath(), key -> new HashMap<>()); Object value = map.get(schemaPath); Object currentValue = column.getMaxValue(); if (value != null) { if (value != currentValue) { partitionColTypeMap.remove(schemaPath); } } else { // the value of a column with primitive type can not be null, // so checks that there are really null value and puts it to the map if (rowCount == column.getNulls()) { map.put(schemaPath, null); } else { map.put(schemaPath, currentValue); } } } else { partitionColTypeMap.remove(schemaPath); } } this.rowCount += rowGroup.getRowCount(); first = false; } }
From source file:org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcherTest.java
@Test public void testPeriodicWatermark() { final MutableLong clock = new MutableLong(); final MutableBoolean isTemporaryIdle = new MutableBoolean(); final List<Watermark> watermarks = new ArrayList<>(); String fakeStream1 = "fakeStream1"; StreamShardHandle shardHandle = new StreamShardHandle(fakeStream1, new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))); TestSourceContext<String> sourceContext = new TestSourceContext<String>() { @Override/*from www. j a v a 2s . co m*/ public void emitWatermark(Watermark mark) { watermarks.add(mark); } @Override public void markAsTemporarilyIdle() { isTemporaryIdle.setTrue(); } }; HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest = new HashMap<>(); final KinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<String>( Collections.singletonList(fakeStream1), sourceContext, new java.util.Properties(), new KinesisDeserializationSchemaWrapper<>( new org.apache.flink.streaming.util.serialization.SimpleStringSchema()), 1, 1, new AtomicReference<>(), new LinkedList<>(), subscribedStreamsToLastSeenShardIdsUnderTest, FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(new HashMap<>())) { @Override protected long getCurrentTimeMillis() { return clock.getValue(); } }; Whitebox.setInternalState(fetcher, "periodicWatermarkAssigner", watermarkAssigner); SequenceNumber seq = new SequenceNumber("fakeSequenceNumber"); // register shards to subsequently emit records int shardIndex = fetcher.registerNewSubscribedShardState(new KinesisStreamShardState( KinesisDataFetcher.convertToStreamShardMetadata(shardHandle), shardHandle, seq)); StreamRecord<String> record1 = new StreamRecord<>(String.valueOf(Long.MIN_VALUE), Long.MIN_VALUE); fetcher.emitRecordAndUpdateState(record1.getValue(), record1.getTimestamp(), shardIndex, seq); Assert.assertEquals(record1, sourceContext.getCollectedOutputs().poll()); fetcher.emitWatermark(); Assert.assertTrue("potential watermark equals previous watermark", watermarks.isEmpty()); StreamRecord<String> record2 = new StreamRecord<>(String.valueOf(1), 1); fetcher.emitRecordAndUpdateState(record2.getValue(), record2.getTimestamp(), shardIndex, seq); Assert.assertEquals(record2, sourceContext.getCollectedOutputs().poll()); fetcher.emitWatermark(); Assert.assertFalse("watermark advanced", watermarks.isEmpty()); Assert.assertEquals(new Watermark(record2.getTimestamp()), watermarks.remove(0)); Assert.assertFalse("not idle", isTemporaryIdle.booleanValue()); // test idle timeout long idleTimeout = 10; // advance clock idleTimeout clock.add(idleTimeout + 1); fetcher.emitWatermark(); Assert.assertFalse("not idle", isTemporaryIdle.booleanValue()); Assert.assertTrue("not idle, no new watermark", watermarks.isEmpty()); // activate idle timeout Whitebox.setInternalState(fetcher, "shardIdleIntervalMillis", idleTimeout); fetcher.emitWatermark(); Assert.assertTrue("idle", isTemporaryIdle.booleanValue()); Assert.assertTrue("idle, no watermark", watermarks.isEmpty()); }
From source file:org.apache.gobblin.writer.ThrottleWriterTest.java
public void testThrottleBytes() throws IOException { DataWriter<Void> writer = mock(DataWriter.class); final MutableLong mockBytes = new MutableLong(); when(writer.bytesWritten()).thenAnswer(new Answer<Long>() { @Override/*from w w w . j ava2s . co m*/ public Long answer(InvocationOnMock invocation) throws Throwable { mockBytes.add(1L); //Delta bytes return mockBytes.getValue(); } }); int parallelism = 2; int bps = 2; DataWriter<Void> throttleWriter = setup(writer, parallelism, bps, ThrottleType.Bytes); int count = 0; long duration = 10L; Stopwatch stopwatch = Stopwatch.createStarted(); while (stopwatch.elapsed(TimeUnit.SECONDS) <= duration) { throttleWriter.writeEnvelope(null); count++; } int expected = (int) (bps * duration); Assert.assertTrue(count <= expected + bps * 2); Assert.assertTrue(count >= expected - bps * 2); }
From source file:org.matsim.contrib.util.LongEnumAdder.java
public LongEnumAdder(Class<K> clazz) { super(clazz); sums = new EnumMap<>(clazz); for (K e : keys) { sums.put(e, new MutableLong()); }//from w w w.j a v a2s.c o m }