List of usage examples for com.google.common.io CountingInputStream CountingInputStream
public CountingInputStream(InputStream in)
From source file:org.apache.tephra.persist.HDFSTransactionStateStorage.java
private TransactionVisibilityState readTransactionVisibilityStateFromInputStream(InputStream in) throws IOException { CountingInputStream countingIn = new CountingInputStream(in); TransactionVisibilityState state = codecProvider.decodeTransactionVisibilityState(countingIn); LOG.info("Read encoded transaction snapshot of {} bytes", countingIn.getCount()); return state; }
From source file:org.dcm4chee.storage.cloud.CloudStorageSystemProvider.java
private void upload(StorageContext ctx, InputStream in, String name, long len) throws IOException { String container = system.getStorageSystemContainer(); BlobStore blobStore = context.getBlobStore(); if (blobStore.blobExists(container, name)) throw new ObjectAlreadyExistsException(system.getStorageSystemPath(), container + '/' + name); CountingInputStream cin = new CountingInputStream(in); Payload payload = new InputStreamPayload(cin); if (len != -1) { payload.getContentMetadata().setContentLength(len); }//www . java 2 s . co m Blob blob = blobStore.blobBuilder(name).payload(payload).build(); String etag = (multipartUploader != null) ? multipartUploader.upload(container, blob) : blobStore.putBlob(container, blob); ctx.setFileSize(cin.getCount()); log.info("Uploaded[uri={}, container={}, name={}, etag={}]", system.getStorageSystemPath(), container, name, etag); }
From source file:com.facebook.buck.json.PythonDslProjectBuildFileParser.java
/** Initialize the parser, starting buck.py. */ private void init() throws IOException { try (SimplePerfEvent.Scope scope = SimplePerfEvent.scope(buckEventBus, PerfEventId.of("ParserInit"))) { ImmutableMap.Builder<String, String> pythonEnvironmentBuilder = ImmutableMap.builder(); // Strip out PYTHONPATH. buck.py manually sets this to include only nailgun. We don't want // to inject nailgun into the parser's PYTHONPATH, so strip that value out. // If we wanted to pass on some environmental PYTHONPATH, we would have to do some actual // merging of this and the BuckConfig's python module search path. pythonEnvironmentBuilder.putAll(Maps.filterKeys(environment, k -> !PYTHONPATH_ENV_VAR_NAME.equals(k))); if (options.getPythonModuleSearchPath().isPresent()) { pythonEnvironmentBuilder.put(PYTHONPATH_ENV_VAR_NAME, options.getPythonModuleSearchPath().get()); }/* w ww . j a va 2 s. c o m*/ ImmutableMap<String, String> pythonEnvironment = pythonEnvironmentBuilder.build(); ProcessExecutorParams params = ProcessExecutorParams.builder().setCommand(buildArgs()) .setEnvironment(pythonEnvironment).build(); LOG.debug("Starting buck.py command: %s environment: %s", params.getCommand(), params.getEnvironment()); buckPyProcess = processExecutor.launchProcess(params); LOG.debug("Started process %s successfully", buckPyProcess); buckPyProcessInput = new CountingInputStream(buckPyProcess.getInputStream()); buckPyProcessJsonGenerator = ObjectMappers.createGenerator(buckPyProcess.getOutputStream()); // We have to wait to create the JsonParser until after we write our // first request, because Jackson "helpfully" synchronously reads // from the InputStream trying to detect whether the encoding is // UTF-8 or UTF-16 as soon as you create a JsonParser: // // https://git.io/vSgnA // // Since buck.py doesn't write any data until after it receives // a query, creating the JsonParser here would hang indefinitely. InputStream stderr = buckPyProcess.getErrorStream(); AtomicInteger numberOfLines = new AtomicInteger(0); AtomicReference<Path> lastPath = new AtomicReference<Path>(); InputStreamConsumer stderrConsumer = new InputStreamConsumer(stderr, (InputStreamConsumer.Handler) line -> { Path path = currentBuildFile.get(); if (!Objects.equals(path, lastPath.get())) { numberOfLines.set(0); lastPath.set(path); } int count = numberOfLines.getAndIncrement(); if (count == 0) { buckEventBus.post(ConsoleEvent.warning("WARNING: Output when parsing %s:", path)); } buckEventBus.post(ConsoleEvent.warning("| %s", line)); }); stderrConsumerTerminationFuture = new FutureTask<>(stderrConsumer); stderrConsumerThread = Threads.namedThread(PythonDslProjectBuildFileParser.class.getSimpleName(), stderrConsumerTerminationFuture); stderrConsumerThread.start(); } }
From source file:org.sonatype.nexus.yum.internal.MetadataProcessor.java
/** * Store primary.xml and update content of repomd.xml accordingly. * * @param repository repository containing primary.xml/repomd.xml * @param repoMDDoc parsed repomd.xml//ww w .ja v a 2s. c o m * @param primaryPath path of primary.xml * @return true if repomd.xml changed */ private static boolean updatePrimaryInRepoMD(final Repository repository, final Document repoMDDoc, final String primaryPath) throws Exception { XPath xPath = XPathFactory.newInstance().newXPath(); String primaryHref = xPath.compile("/repomd/data[@type='primary']/location/@href").evaluate(repoMDDoc); if (!Objects.equals(primaryPath, primaryHref)) { log.debug("Updating 'primary' data entry in {}:repomd.xml", repository.getId()); Element primaryEl = (Element) xPath.compile("/repomd/data[@type='primary']").evaluate(repoMDDoc, NODE); StorageFileItem primaryItem = (StorageFileItem) repository.retrieveItem(false, new ResourceStoreRequest("/" + primaryPath)); try (InputStream in = primaryItem.getInputStream(); CountingInputStream cis = new CountingInputStream( new GZIPInputStream(new BufferedInputStream(in)))) { HashCode checksum = Hashes.hash(Hashing.sha256(), cis); primaryEl.getElementsByTagName("open-checksum").item(0).setTextContent(checksum.toString()); primaryEl.getElementsByTagName("open-size").item(0).setTextContent(String.valueOf(cis.getCount())); } primaryItem = (StorageFileItem) repository.retrieveItem(false, new ResourceStoreRequest("/" + primaryPath)); try (InputStream in = primaryItem.getInputStream(); CountingInputStream cis = new CountingInputStream(new BufferedInputStream(in))) { HashCode checksum = Hashes.hash(Hashing.sha256(), cis); primaryEl.getElementsByTagName("checksum").item(0).setTextContent(checksum.toString()); primaryEl.getElementsByTagName("size").item(0).setTextContent(String.valueOf(cis.getCount())); } ((Element) primaryEl.getElementsByTagName("location").item(0)).setAttribute("href", primaryPath); return true; } return false; }
From source file:org.apache.jackrabbit.oak.plugins.tika.TextExtractor.java
private String parseStringValue(ByteSource byteSource, Metadata metadata, String path) { WriteOutContentHandler handler = new WriteOutContentHandler(maxExtractedLength); long start = System.currentTimeMillis(); long size = 0; try {//www. j a va 2 s . c om CountingInputStream stream = new CountingInputStream(new LazyInputStream(byteSource)); try { tika.getParser().parse(stream, handler, metadata, new ParseContext()); } finally { size = stream.getCount(); stream.close(); } } catch (LinkageError e) { // Capture and ignore errors caused by extraction libraries // not being present. This is equivalent to disabling // selected media types in configuration, so we can simply // ignore these errors. } catch (Throwable t) { // Capture and report any other full text extraction problems. // The special STOP exception is used for normal termination. if (!handler.isWriteLimitReached(t)) { parserErrorCount.incrementAndGet(); parserError.debug("Failed to extract text from a binary property: " + path + " This is a fairly common case, and nothing to" + " worry about. The stack trace is included to" + " help improve the text extraction feature.", t); return ERROR_TEXT; } } String result = handler.toString(); timeTaken.addAndGet(System.currentTimeMillis() - start); if (size > 0) { extractedTextSize.addAndGet(result.length()); extractionCount.incrementAndGet(); totalSizeRead.addAndGet(size); return result; } return null; }
From source file:org.apache.beam.sdk.testing.CoderProperties.java
@VisibleForTesting static <T> T decode(Coder<T> coder, Coder.Context context, byte[] bytes) throws CoderException, IOException { @SuppressWarnings("unchecked") Coder<T> deserializedCoder = SerializableUtils.clone(coder); byte[] buffer; if (context == Coder.Context.NESTED) { buffer = new byte[bytes.length + 1]; System.arraycopy(bytes, 0, buffer, 0, bytes.length); buffer[bytes.length] = 1;//from ww w . j a v a 2s.co m } else { buffer = bytes; } CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(buffer)); T value = deserializedCoder.decode(new UnownedInputStream(cis), context); assertThat("consumed bytes equal to encoded bytes", cis.getCount(), equalTo((long) bytes.length)); return value; }
From source file:org.pgptool.gui.encryption.implpgp.EncryptionServicePgpImpl.java
@Override public void decrypt(String sourceFile, String targetFile, PasswordDeterminedForKey keyAndPassword, ProgressHandler optionalProgressHandler, OutputStreamSupervisor optionalOutputStreamSupervisor) throws InvalidPasswordException, UserRequestedCancellationException { OutputStreamSupervisor outputStreamSupervisor = optionalOutputStreamSupervisor != null ? optionalOutputStreamSupervisor : new OutputStreamSupervisorImpl(); log.debug("Decrypting " + sourceFile); Updater progress = null;// w ww.ja v a 2 s . c o m BigInteger sourceSize = BigInteger.valueOf(new File(sourceFile).length()); if (optionalProgressHandler != null) { progress = Progress.create("action.decrypt", optionalProgressHandler); progress.updateStepInfo("progress.preparingKeys", FilenameUtils.getName(sourceFile)); progress.updateTotalSteps(sourceSize); } Key decryptionKey = keyAndPassword.getMatchedKey(); String passphrase = keyAndPassword.getPassword(); Preconditions.checkArgument(StringUtils.hasText(sourceFile) && new File(sourceFile).exists(), "Source file name must be correct"); Preconditions.checkArgument(StringUtils.hasText(targetFile), "Target file name must be provided"); Preconditions.checkArgument(decryptionKey != null, "decryption key must be provided"); Preconditions.checkArgument(StringUtils.hasText(passphrase), "Passphrase must be provided"); InputStream in = null; try { PGPSecretKey secretKey = KeyDataPgp.get(decryptionKey) .findSecretKeyById(keyAndPassword.getDecryptionKeyId()); PGPPrivateKey privateKey = getPrivateKey(passphrase, secretKey); CountingInputStream countingStream = new CountingInputStream(new FileInputStream(sourceFile)); in = new BufferedInputStream(countingStream); PGPPublicKeyEncryptedData publicKeyEncryptedData = getPublicKeyEncryptedDataByKeyId(in, secretKey); OutputStream outputStream = outputStreamSupervisor.get(targetFile); decryptStream(publicKeyEncryptedData, privateKey, outputStream, progress, countingStream); if (optionalProgressHandler != null) { // NOTE: The problem with decryption is that BC doesn't provide API to get // compressed+encrypted file size so it's hard to report progress precisely. We // do our best, but still it's very approximate. So that's why we need to // explicitly set 100% after operation was completed progress.updateTotalSteps(sourceSize); } } catch (Throwable t) { File fileToDelete = new File(targetFile); if (fileToDelete.exists() && !fileToDelete.delete()) { log.warn("Failed to delete file after failed decryption: " + targetFile); } Throwables.throwIfInstanceOf(t, InvalidPasswordException.class); Throwables.throwIfInstanceOf(t, UserRequestedCancellationException.class); log.error("Decryption failed", t); throw new RuntimeException("Decryption failed", t); } finally { IoStreamUtils.safeClose(in); } }
From source file:org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditor.java
private String parseStringValue0(Blob v, Metadata metadata, String path) { WriteOutContentHandler handler = new WriteOutContentHandler(context.getDefinition().getMaxExtractLength()); long start = System.currentTimeMillis(); long bytesRead = 0; try {/*from ww w. ja v a2 s .c o m*/ CountingInputStream stream = new CountingInputStream(new LazyInputStream(new BlobByteSource(v))); try { context.getParser().parse(stream, handler, metadata, new ParseContext()); } finally { bytesRead = stream.getCount(); stream.close(); } } catch (LinkageError e) { // Capture and ignore errors caused by extraction libraries // not being present. This is equivalent to disabling // selected media types in configuration, so we can simply // ignore these errors. } catch (Throwable t) { // Capture and report any other full text extraction problems. // The special STOP exception is used for normal termination. if (!handler.isWriteLimitReached(t)) { log.debug("[{}] Failed to extract text from a binary property: {}." + " This is a fairly common case, and nothing to" + " worry about. The stack trace is included to" + " help improve the text extraction feature.", getIndexName(), path, t); context.getExtractedTextCache().put(v, ExtractedText.ERROR); return TEXT_EXTRACTION_ERROR; } } String result = handler.toString(); if (bytesRead > 0) { context.recordTextExtractionStats(System.currentTimeMillis() - start, bytesRead, result.length()); } context.getExtractedTextCache().put(v, new ExtractedText(ExtractionResult.SUCCESS, result)); return result; }