List of usage examples for java.util.concurrent.atomic AtomicInteger addAndGet
public final int addAndGet(int delta)
From source file:xc.mst.repo.RepositoryDAO.java
public boolean checkOutsideRange(String sql, AtomicInteger tally, AtomicInteger numMatching) { // could LIMIT accomplish the same thing as explain ? // no - I dont think so, because if the answer is zero - limit takes a looooong time List<Map<String, Object>> records = this.jdbcTemplate.queryForList("explain " + sql); BigInteger rows2examine = (BigInteger) records.get(0).get("rows"); LOG.debug("rows: " + rows2examine); if (rows2examine == null) { return false; } else if (rows2examine.intValue() > getEstimateCompleteListSizeThreshold()) { if (tally != null) { tally.addAndGet(2); }/*from www . j a va2s.co m*/ if (numMatching != null) { numMatching.set(rows2examine.intValue()); } } else { int exactCount = this.jdbcTemplate.queryForInt(sql); if (exactCount > 0) { if (tally != null) { tally.addAndGet(1); } if (numMatching != null) { numMatching.set(exactCount); } } } if (tally != null) LOG.debug("tally: " + tally.get()); if (numMatching != null) LOG.debug("numMatching: " + numMatching.get()); return true; }
From source file:com.btoddb.fastpersitentqueue.FpqIT.java
@Test public void testThreading() throws Exception { final int numEntries = 1000; final int numPushers = 4; final int numPoppers = 4; final int entrySize = 1000; fpq1.setMaxTransactionSize(2000);/*from ww w. ja v a 2 s . c o m*/ final int popBatchSize = 100; fpq1.setMaxMemorySegmentSizeInBytes(10000000); fpq1.setMaxJournalFileSize(10000000); fpq1.setMaxJournalDurationInMs(30000); fpq1.setFlushPeriodInMs(1000); fpq1.setNumberOfFlushWorkers(4); final Random pushRand = new Random(1000L); final Random popRand = new Random(1000000L); final AtomicInteger pusherFinishCount = new AtomicInteger(); final AtomicInteger numPops = new AtomicInteger(); final AtomicLong counter = new AtomicLong(); final AtomicLong pushSum = new AtomicLong(); final AtomicLong popSum = new AtomicLong(); fpq1.init(); ExecutorService execSrvc = Executors.newFixedThreadPool(numPushers + numPoppers); Set<Future> futures = new HashSet<Future>(); // start pushing for (int i = 0; i < numPushers; i++) { Future future = execSrvc.submit(new Runnable() { @Override public void run() { for (int i = 0; i < numEntries; i++) { try { long x = counter.getAndIncrement(); pushSum.addAndGet(x); ByteBuffer bb = ByteBuffer.wrap(new byte[entrySize]); bb.putLong(x); fpq1.beginTransaction(); fpq1.push(bb.array()); fpq1.commit(); if ((x + 1) % 500 == 0) { System.out.println("pushed ID = " + x); } Thread.sleep(pushRand.nextInt(5)); } catch (Exception e) { e.printStackTrace(); } } pusherFinishCount.incrementAndGet(); } }); futures.add(future); } // start popping for (int i = 0; i < numPoppers; i++) { Future future = execSrvc.submit(new Runnable() { @Override public void run() { while (pusherFinishCount.get() < numPushers || !fpq1.isEmpty()) { try { fpq1.beginTransaction(); try { Collection<FpqEntry> entries = fpq1.pop(popBatchSize); if (null == entries) { Thread.sleep(100); continue; } for (FpqEntry entry : entries) { ByteBuffer bb = ByteBuffer.wrap(entry.getData()); popSum.addAndGet(bb.getLong()); if (entry.getId() % 500 == 0) { System.out.println("popped ID = " + entry.getId()); } } numPops.addAndGet(entries.size()); fpq1.commit(); entries.clear(); } finally { if (fpq1.isTransactionActive()) { fpq1.rollback(); } } Thread.sleep(popRand.nextInt(10)); } catch (Exception e) { e.printStackTrace(); } } } }); futures.add(future); } boolean finished = false; while (!finished) { try { for (Future f : futures) { f.get(); } finished = true; } catch (InterruptedException e) { // ignore Thread.interrupted(); } } assertThat(numPops.get(), is(numEntries * numPushers)); assertThat(fpq1.getNumberOfEntries(), is(0L)); assertThat(pushSum.get(), is(popSum.get())); assertThat(fpq1.getMemoryMgr().getNumberOfActiveSegments(), is(1)); assertThat(fpq1.getMemoryMgr().getSegments(), hasSize(1)); assertThat(fpq1.getJournalMgr().getJournalFiles().entrySet(), hasSize(1)); assertThat(FileUtils.listFiles(fpq1.getPagingDirectory(), TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE), is(empty())); assertThat( FileUtils.listFiles(fpq1.getJournalDirectory(), TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE), hasSize(1)); }
From source file:org.neo4j.io.pagecache.PageCacheTest.java
@Test public void pagedFileFlushAndForceMustQueryTheGivenIOPSLimiter() throws Exception { int pagesToDirty = 10_000; PageCache cache = getPageCache(fs, nextPowerOf2(pagesToDirty), pageCachePageSize, PageCacheTracer.NULL); PagedFile pf = cache.map(file("a"), filePageSize); // Dirty a bunch of data dirtyManyPages(pf, pagesToDirty);//from w ww.j ava2 s.c om AtomicInteger callbackCounter = new AtomicInteger(); AtomicInteger ioCounter = new AtomicInteger(); pf.flushAndForce((previousStamp, recentlyCompletedIOs, swapper) -> { ioCounter.addAndGet(recentlyCompletedIOs); return callbackCounter.getAndIncrement(); }); pf.close(); assertThat(callbackCounter.get(), greaterThan(0)); assertThat(ioCounter.get(), greaterThanOrEqualTo(pagesToDirty - 30)); // -30 because of the eviction thread }
From source file:org.neo4j.io.pagecache.PageCacheTest.java
@Test public void pageCacheFlushAndForceMustQueryTheGivenIOPSLimiter() throws Exception { int pagesToDirty = 10_000; PageCache cache = getPageCache(fs, nextPowerOf2(2 * pagesToDirty), pageCachePageSize, PageCacheTracer.NULL); PagedFile pfA = cache.map(existingFile("a"), filePageSize); PagedFile pfB = cache.map(existingFile("b"), filePageSize); dirtyManyPages(pfA, pagesToDirty);//from w w w.j a v a 2 s . c o m dirtyManyPages(pfB, pagesToDirty); AtomicInteger callbackCounter = new AtomicInteger(); AtomicInteger ioCounter = new AtomicInteger(); cache.flushAndForce((previousStamp, recentlyCompletedIOs, swapper) -> { ioCounter.addAndGet(recentlyCompletedIOs); return callbackCounter.getAndIncrement(); }); pfA.close(); pfB.close(); assertThat(callbackCounter.get(), greaterThan(0)); assertThat(ioCounter.get(), greaterThanOrEqualTo(pagesToDirty * 2 - 30)); // -30 because of the eviction thread }
From source file:org.alfresco.repo.activities.feed.FeedNotifierImpl.java
private void executeInternal(final int repeatIntervalMins) { final String emailTemplateRef = getEmailTemplateRef(); if (emailTemplateRef == null) { return;/*w ww . j av a2 s.c o m*/ } final String shareUrl = UrlUtil.getShareUrl(sysAdminParams); if (logger.isDebugEnabled()) { logger.debug("Share URL configured as: " + shareUrl); } final AtomicInteger userCnt = new AtomicInteger(0); final AtomicInteger feedEntryCnt = new AtomicInteger(0); final long startTime = System.currentTimeMillis(); // local cache for this execution final Map<String, String> siteNames = new ConcurrentHashMap<String, String>(10); try { final String currentUser = AuthenticationUtil.getRunAsUser(); final String tenantDomain = TenantUtil.getCurrentDomain(); // process the feeds using the batch processor {@link BatchProcessor} BatchProcessor.BatchProcessWorker<PersonInfo> worker = new BatchProcessor.BatchProcessWorker<PersonInfo>() { public String getIdentifier(final PersonInfo person) { StringBuilder sb = new StringBuilder("Person "); sb.append(person.getUserName()); return sb.toString(); } public void beforeProcess() throws Throwable { AuthenticationUtil.pushAuthentication(); AuthenticationUtil.setFullyAuthenticatedUser(currentUser); } public void afterProcess() throws Throwable { AuthenticationUtil.popAuthentication(); } public void process(final PersonInfo person) throws Throwable { final RetryingTransactionHelper txHelper = transactionService.getRetryingTransactionHelper(); txHelper.setMaxRetries(0); TenantUtil.runAsTenant(new TenantRunAsWork<Void>() { @Override public Void doWork() throws Exception { txHelper.doInTransaction(new RetryingTransactionCallback<Void>() { public Void execute() throws Throwable { processInternal(person); return null; } }, false, true); return null; } }, tenantDomain); } private void processInternal(final PersonInfo person) throws Exception { final NodeRef personNodeRef = person.getNodeRef(); try { Pair<Integer, Long> result = userNotifier.notifyUser(personNodeRef, MSG_EMAIL_SUBJECT, new Object[] { ModelUtil.getProductName(repoAdminService) }, siteNames, shareUrl, repeatIntervalMins, emailTemplateRef); if (result != null) { int entryCnt = result.getFirst(); final long maxFeedId = result.getSecond(); Long currentMaxFeedId = (Long) nodeService.getProperty(personNodeRef, ContentModel.PROP_EMAIL_FEED_ID); if ((currentMaxFeedId == null) || (currentMaxFeedId < maxFeedId)) { nodeService.setProperty(personNodeRef, ContentModel.PROP_EMAIL_FEED_ID, maxFeedId); } userCnt.incrementAndGet(); feedEntryCnt.addAndGet(entryCnt); } } catch (InvalidNodeRefException inre) { // skip this person - eg. no longer exists ? logger.warn( "Skip feed notification for user (" + personNodeRef + "): " + inre.getMessage()); } } }; // grab people for the batch processor in chunks of size batchSize BatchProcessWorkProvider<PersonInfo> provider = new BatchProcessWorkProvider<PersonInfo>() { private int skip = 0; private int maxItems = batchSize; private boolean hasMore = true; @Override public int getTotalEstimatedWorkSize() { return personService.countPeople(); } @Override public Collection<PersonInfo> getNextWork() { if (!hasMore) { return Collections.emptyList(); } PagingResults<PersonInfo> people = personService.getPeople(null, null, null, new PagingRequest(skip, maxItems)); List<PersonInfo> page = people.getPage(); skip += page.size(); hasMore = people.hasMoreItems(); return page; } }; final RetryingTransactionHelper txHelper = transactionService.getRetryingTransactionHelper(); txHelper.setMaxRetries(0); new BatchProcessor<PersonInfo>("FeedNotifier", txHelper, provider, numThreads, batchSize, applicationContext, logger, 100).process(worker, true); } catch (Throwable e) { // If the VM is shutting down, then ignore if (vmShutdownListener.isVmShuttingDown()) { // Ignore } else { logger.error("Exception during notification of feeds", e); } } finally { int count = userCnt.get(); int entryCount = feedEntryCnt.get(); // assume sends are synchronous - hence bump up to last max feed id if (count > 0) { if (logger.isInfoEnabled()) { // TODO i18n of info message StringBuilder sb = new StringBuilder(); sb.append("Notified ").append(userCnt).append(" user").append(count != 1 ? "s" : ""); sb.append(" of ").append(feedEntryCnt).append(" activity feed entr") .append(entryCount != 1 ? "ies" : "y"); sb.append(" (in ").append(System.currentTimeMillis() - startTime).append(" msecs)"); logger.info(sb.toString()); } } else { if (logger.isTraceEnabled()) { logger.trace("Nothing to send since no new user activities found"); } } } }
From source file:org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC.java
@SuppressWarnings("checkstyle:methodlength") @Override/* w w w .ja va 2 s .c o m*/ public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException { final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior(); logger.debug("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle); Pair<Short, Short> shardNumAndBaseShard = getShardNumAndBaseShard(); short shardNum = shardNumAndBaseShard.getFirst(); short cuboidBaseShard = shardNumAndBaseShard.getSecond(); int totalShards = cubeSeg.getTotalShards(); ByteString scanRequestByteString = null; ByteString rawScanByteString = null; // primary key (also the 0th column block) is always selected final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0); // globally shared connection, does not require close final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl()); final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList(); List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks); for (List<Integer> list : hbaseColumnsToGT) { hbaseColumnsToGTIntList.add(IntList.newBuilder().addAllInts(list).build()); } //TODO: raw scan can be constructed at region side to reduce traffic List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks); int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE; while (true) { try { ByteBuffer rawScanBuffer = ByteBuffer.allocate(rawScanBufferSize); BytesUtil.writeVInt(rawScans.size(), rawScanBuffer); for (RawScan rs : rawScans) { RawScan.serializer.serialize(rs, rawScanBuffer); } rawScanBuffer.flip(); rawScanByteString = HBaseZeroCopyByteString.wrap(rawScanBuffer.array(), rawScanBuffer.position(), rawScanBuffer.limit()); break; } catch (BufferOverflowException boe) { logger.info("Buffer size {} cannot hold the raw scans, resizing to 4 times", rawScanBufferSize); rawScanBufferSize *= 4; } } scanRequest.setGTScanRanges(Lists.<GTScanRange>newArrayList());//since raw scans are sent to coprocessor, we don't need to duplicate sending it int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE; while (true) { try { ByteBuffer buffer = ByteBuffer.allocate(scanRequestBufferSize); GTScanRequest.serializer.serialize(scanRequest, buffer); buffer.flip(); scanRequestByteString = HBaseZeroCopyByteString.wrap(buffer.array(), buffer.position(), buffer.limit()); break; } catch (BufferOverflowException boe) { logger.info("Buffer size {} cannot hold the scan request, resizing to 4 times", scanRequestBufferSize); scanRequestBufferSize *= 4; } } logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size()); logger.info( "The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg, rawScans.size()); for (RawScan rs : rawScans) { logScan(rs, cubeSeg.getStorageLocationIdentifier()); } logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size()); final AtomicInteger totalScannedCount = new AtomicInteger(0); final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum); // KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries // for different cubes until redeployment of coprocessor jar. final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); final boolean compressionResult = kylinConfig.getCompressionResult(); final CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder(); builder.setGtScanRequest(scanRequestByteString).setHbaseRawScan(rawScanByteString); for (IntList intList : hbaseColumnsToGTIntList) { builder.addHbaseColumnsToGT(intList); } builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize()); builder.setBehavior(toggle); builder.setStartTime(System.currentTimeMillis()); builder.setTimeout(epResultItr.getTimeout()); builder.setKylinProperties(kylinConfig.getConfigAsString()); for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) { executorService.submit(new Runnable() { @Override public void run() { final String logHeader = "<sub-thread for GTScanRequest " + Integer.toHexString(System.identityHashCode(scanRequest)) + "> "; final boolean[] abnormalFinish = new boolean[1]; try { HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool()); final CubeVisitRequest request = builder.build(); final byte[] startKey = epRange.getFirst(); final byte[] endKey = epRange.getSecond(); table.coprocessorService(CubeVisitService.class, startKey, endKey, // new Batch.Call<CubeVisitService, CubeVisitResponse>() { public CubeVisitResponse call(CubeVisitService rowsService) throws IOException { ServerRpcController controller = new ServerRpcController(); BlockingRpcCallback<CubeVisitResponse> rpcCallback = new BlockingRpcCallback<>(); rowsService.visitCube(controller, request, rpcCallback); CubeVisitResponse response = rpcCallback.get(); if (controller.failedOnException()) { throw controller.getFailedOn(); } return response; } }, new Batch.Callback<CubeVisitResponse>() { @Override public void update(byte[] region, byte[] row, CubeVisitResponse result) { if (region == null) return; totalScannedCount.addAndGet(result.getStats().getScannedRowCount()); logger.info(logHeader + getStatsString(region, result)); if (result.getStats().getNormalComplete() != 1) { abnormalFinish[0] = true; return; } try { if (compressionResult) { epResultItr .append(CompressionUtils.decompress(HBaseZeroCopyByteString .zeroCopyGetBytes(result.getCompressedRows()))); } else { epResultItr.append(HBaseZeroCopyByteString .zeroCopyGetBytes(result.getCompressedRows())); } } catch (IOException | DataFormatException e) { throw new RuntimeException(logHeader + "Error when decompressing", e); } } }); } catch (Throwable ex) { logger.error(logHeader + "Error when visiting cubes by endpoint", ex); // double log coz the query thread may already timeout epResultItr.notifyCoprocException(ex); return; } if (abnormalFinish[0]) { Throwable ex = new RuntimeException(logHeader + "The coprocessor thread stopped itself due to scan timeout, failing current query..."); logger.error(logHeader + "Error when visiting cubes by endpoint", ex); // double log coz the query thread may already timeout epResultItr.notifyCoprocException(ex); return; } } }); } return new EndpointResultsAsGTScanner(fullGTInfo, epResultItr, scanRequest.getColumns(), totalScannedCount.get()); }
From source file:io.warp10.continuum.gts.GTSHelper.java
/** * Compute conditional probabilities given a GTS considering the values as the concatenation * of given events plus the event for which we want to compute the probability, separated by 'separator' * /* w w w . j a v a2 s . c om*/ * If 'sepearator' is null then we simply compute the probability of values instead of conditional probabilities * * @param gts * @param separator * @return */ public static GeoTimeSerie cprob(GeoTimeSerie gts, String separator) throws WarpScriptException { Map<Object, AtomicInteger> histogram = new HashMap<Object, AtomicInteger>(); GeoTimeSerie prob = gts.cloneEmpty(); if (null == separator) { long total = 0L; for (int i = 0; i < gts.values; i++) { Object value = GTSHelper.valueAtIndex(gts, i); AtomicInteger count = histogram.get(value); if (null == count) { count = new AtomicInteger(0); histogram.put(value, count); } count.addAndGet(1); total++; } for (int i = 0; i < gts.values; i++) { long timestamp = GTSHelper.tickAtIndex(gts, i); long geoxppoint = GTSHelper.locationAtIndex(gts, i); long elevation = GTSHelper.elevationAtIndex(gts, i); Object value = GTSHelper.valueAtIndex(gts, i); double p = histogram.get(value).doubleValue() / total; GTSHelper.setValue(prob, timestamp, geoxppoint, elevation, p, false); } return prob; } // // Sort 'gts' by value so we group the 'given' events by common set of values // GTSHelper.valueSort(gts); int idx = 0; while (idx < gts.values) { // // Extract 'given' events // Object val = GTSHelper.valueAtIndex(gts, idx); if (!(val instanceof String)) { throw new WarpScriptException( "Can only compute conditional probabilities for String Geo Time Series."); } int lastsep = val.toString().lastIndexOf(separator); if (-1 == lastsep) { throw new WarpScriptException("Separator not found, unable to isolate given events."); } String given = val.toString().substring(0, lastsep); histogram.clear(); long total = 0; int subidx = idx; while (subidx < gts.values) { val = GTSHelper.valueAtIndex(gts, subidx); lastsep = val.toString().lastIndexOf(separator); if (-1 == lastsep) { throw new WarpScriptException("Separator not found, unable to isolate given events."); } String givenEvents = val.toString().substring(0, lastsep); if (!givenEvents.equals(given)) { break; } String event = val.toString().substring(lastsep + separator.length()).trim(); AtomicInteger count = histogram.get(event); if (null == count) { count = new AtomicInteger(0); histogram.put(event, count); } count.addAndGet(1); total++; subidx++; } for (int i = idx; i < subidx; i++) { val = GTSHelper.valueAtIndex(gts, i); lastsep = val.toString().lastIndexOf(separator); String event = val.toString().substring(lastsep + separator.length()); long timestamp = GTSHelper.tickAtIndex(gts, i); long location = GTSHelper.locationAtIndex(gts, i); long elevation = GTSHelper.elevationAtIndex(gts, i); double p = histogram.get(event).doubleValue() / total; GTSHelper.setValue(prob, timestamp, location, elevation, p, false); } idx = subidx; } return prob; }