List of usage examples for com.google.common.io FileBackedOutputStream FileBackedOutputStream
public FileBackedOutputStream(int fileThreshold)
From source file:ddf.catalog.transformer.input.tika.TikaInputTransformer.java
@Override public Metacard transform(InputStream input, String id) throws IOException, CatalogTransformerException { LOGGER.debug("Transforming input stream using Tika."); if (input == null) { throw new CatalogTransformerException("Cannot transform null input."); }/*from ww w. ja v a 2 s . co m*/ try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(1000000)) { try { IOUtils.copy(input, fileBackedOutputStream); } catch (IOException e) { throw new CatalogTransformerException("Could not copy bytes of content message.", e); } Parser parser = new AutoDetectParser(); ToXMLContentHandler handler = new ToXMLContentHandler(); TikaMetadataExtractor tikaMetadataExtractor = new TikaMetadataExtractor(parser, handler); Metadata metadata; try (InputStream inputStreamCopy = fileBackedOutputStream.asByteSource().openStream()) { metadata = tikaMetadataExtractor.parseMetadata(inputStreamCopy, new ParseContext()); } String metadataText = handler.toString(); if (templates != null) { metadataText = transformToXml(metadataText); } Metacard metacard = MetacardCreator.createBasicMetacard(metadata, id, metadataText); String metacardContentType = metacard.getContentTypeName(); if (StringUtils.startsWith(metacardContentType, "image")) { try (InputStream inputStreamCopy = fileBackedOutputStream.asByteSource().openStream()) { createThumbnail(inputStreamCopy, metacard); } } LOGGER.debug("Finished transforming input stream using Tika."); return metacard; } }
From source file:org.codice.ddf.catalog.transformer.zip.ZipCompression.java
private InputStream createZip(SourceResponse sourceResponse, String transformerId) throws CatalogTransformerException { ServiceReference<MetacardTransformer> serviceRef = getTransformerServiceReference(transformerId); MetacardTransformer transformer = bundleContext.getService(serviceRef); String extension = getFileExtensionFromService(serviceRef); if (StringUtils.isNotBlank(extension)) { extension = "." + extension; }/*from w ww. j av a 2s. c om*/ try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(BUFFER_SIZE); ZipOutputStream zipOutputStream = new ZipOutputStream(fileBackedOutputStream)) { for (Result result : sourceResponse.getResults()) { Metacard metacard = result.getMetacard(); BinaryContent binaryContent = getTransformedMetacard(metacard, Collections.emptyMap(), transformer); if (binaryContent != null) { ZipEntry entry = new ZipEntry(METACARD_PATH + metacard.getId() + extension); zipOutputStream.putNextEntry(entry); zipOutputStream.write(binaryContent.getByteArray()); zipOutputStream.closeEntry(); } else { LOGGER.debug("Metacard with id [{}] was not added to zip file", metacard.getId()); } } return fileBackedOutputStream.asByteSource().openStream(); } catch (IOException e) { throw new CatalogTransformerException("An error occurred while initializing or closing output stream", e); } }
From source file:org.openqa.selenium.remote.NewSessionPayload.java
private NewSessionPayload(Reader source) throws IOException { // Dedicate up to 10% of all RAM or 20% of available RAM (whichever is smaller) to storing this // payload.// ww w. j av a 2 s. c om int threshold = (int) Math.min(Integer.MAX_VALUE, Math.min(Runtime.getRuntime().freeMemory() / 5, Runtime.getRuntime().maxMemory() / 10)); backingStore = new FileBackedOutputStream(threshold); try (Writer writer = new OutputStreamWriter(backingStore, UTF_8)) { CharStreams.copy(source, writer); } ImmutableSet.Builder<CapabilitiesFilter> adapters = ImmutableSet.builder(); ServiceLoader.load(CapabilitiesFilter.class).forEach(adapters::add); adapters.add(new ChromeFilter()).add(new EdgeFilter()).add(new FirefoxFilter()) .add(new InternetExplorerFilter()).add(new OperaFilter()).add(new SafariFilter()); this.adapters = adapters.build(); ImmutableSet.Builder<CapabilityTransform> transforms = ImmutableSet.builder(); ServiceLoader.load(CapabilityTransform.class).forEach(transforms::add); transforms.add(new ProxyTransform()).add(new StripAnyPlatform()).add(new W3CPlatformNameNormaliser()); this.transforms = transforms.build(); ImmutableSet.Builder<Dialect> dialects = ImmutableSet.builder(); if (getOss() != null) { dialects.add(Dialect.OSS); } if (getAlwaysMatch() != null || getFirstMatches() != null) { dialects.add(Dialect.W3C); } this.dialects = dialects.build(); validate(); }
From source file:org.codice.alliance.transformer.video.MpegTsInputTransformer.java
@Override public Metacard transform(InputStream inputStream, final String id) throws IOException, CatalogTransformerException { LOGGER.info("processing video input for id = {}", id); try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(FILE_THRESHOLD)) { populateFileBackedOutputStream(inputStream, fileBackedOutputStream); MetacardImpl metacard = extractInnerTransformerMetadata(id, fileBackedOutputStream); extractStanag4609Metadata(metacard, fileBackedOutputStream); return metacard; }//w w w . ja v a 2s . c o m }
From source file:ddf.catalog.resource.download.ReliableResourceDownloader.java
public ResourceResponse setupDownload(Metacard metacard, DownloadStatusInfo downloadStatusInfo) { Resource resource = resourceResponse.getResource(); MimeType mimeType = resource.getMimeType(); String resourceName = resource.getName(); fbos = new FileBackedOutputStream(DEFAULT_FILE_BACKED_OUTPUT_STREAM_THRESHOLD); countingFbos = new CountingOutputStream(fbos); streamReadByClient = new ReliableResourceInputStream(fbos, countingFbos, downloadState, downloadIdentifier, resourceResponse);//from ww w . ja v a2s. co m this.metacard = metacard; // Create new ResourceResponse to return that will encapsulate the // ReliableResourceInputStream that will be read by the client simultaneously as the product // is cached to disk (if caching is enabled) ResourceImpl newResource = new ResourceImpl(streamReadByClient, mimeType, resourceName); resourceResponse = new ResourceResponseImpl(resourceResponse.getRequest(), resourceResponse.getProperties(), newResource); // Get handle to retrieved product's InputStream resourceInputStream = resource.getInputStream(); eventListener.setDownloadMap(downloadIdentifier, resourceResponse); downloadStatusInfo.addDownloadInfo(downloadIdentifier, this, resourceResponse); if (downloaderConfig.isCacheEnabled()) { CacheKey keyMaker = null; String key = null; try { keyMaker = new CacheKey(metacard, resourceResponse.getRequest()); key = keyMaker.generateKey(); } catch (IllegalArgumentException e) { LOGGER.info("Cannot create cache key for resource with metacard ID = {}", metacard.getId()); return resourceResponse; } if (!resourceCache.isPending(key)) { // Fully qualified path to cache file that will be written to. // Example: // <INSTALL-DIR>/data/product-cache/<source-id>-<metacard-id> // <INSTALL-DIR>/data/product-cache/ddf.distribution-abc123 filePath = FilenameUtils.concat(resourceCache.getProductCacheDirectory(), key); reliableResource = new ReliableResource(key, filePath, mimeType, resourceName, metacard); resourceCache.addPendingCacheEntry(reliableResource); try { fos = FileUtils.openOutputStream(new File(filePath)); doCaching = true; this.downloadState.setCacheEnabled(true); } catch (IOException e) { LOGGER.info("Unable to open cache file {} - no caching will be done.", filePath); } } else { LOGGER.debug("Cache key {} is already pending caching", key); } } return resourceResponse; }
From source file:org.fcrepo.triplegenerators.tei.TeiTripleGenerator.java
/** * @param resource An {@link InputStream} with TEI XML. * @return A {@code byte[]} of RDF/XML.// w w w . j av a 2 s. co m * @throws IOException * @throws TransformerException */ private byte[] createRDFXML(final InputStream resource) throws IOException, TransformerException { final Source resourceSource = new StreamSource(resource); try (final FileBackedOutputStream addIdsResultStream = new FileBackedOutputStream(1024 * 1024)) { final Result addIdsResult = new StreamResult(addIdsResultStream); addIdsXform.transform(resourceSource, addIdsResult); LOGGER.debug("Added XML IDs to TEI."); try (final InputStream tei2RdfSourceStream = addIdsResultStream.getSupplier().getInput()) { final Source tei2RdfSource = new StreamSource(tei2RdfSourceStream); final StreamResult tei2RdfResult = new StreamResult(new StringWriter()); tei2RdfXform.transform(tei2RdfSource, tei2RdfResult); LOGGER.debug("Created RDF/XML from TEI: \n{}", tei2RdfResult.getWriter().toString()); return tei2RdfResult.getWriter().toString().getBytes(); } } }
From source file:org.codice.ddf.security.crl.generator.CrlGenerator.java
/** Pulls down the CRL file provided via URL and writes it locally. */ @Override/* www . j a v a 2s. c o m*/ public synchronized void run() { if (!crlByUrlEnabled || crlLocationUrl == null) { return; } if (!crlLocationUrl.startsWith(HTTPS)) { postErrorEvent("The provided URL was not an HTTPS URL."); return; } Object entity = getRemoteCrl(); if (!(entity instanceof InputStream)) { postErrorEvent("Unable to download the remote CRL."); return; } FileBackedOutputStream fileBackedOutputStream = null; try { // Read the response content and get the byte source ByteSource byteSource; try (InputStream entityStream = (InputStream) entity) { fileBackedOutputStream = new FileBackedOutputStream(FILE_BACKED_STREAM_THRESHOLD); IOUtils.copy(entityStream, fileBackedOutputStream); fileBackedOutputStream.close(); byteSource = fileBackedOutputStream.asByteSource(); } File crlFile = getCrlFile(byteSource); // Verify that the CRL is valid if (!crlIsValid(byteSource)) { postErrorEvent("An error occurred while validating the CRL."); return; } writeCrlToFile(byteSource, crlFile); } catch (IOException e) { LOGGER.warn("Unable to copy CRL to local CRL. {}", e.getMessage()); postErrorEvent("An error occurred while downloading the CRL."); } finally { // Cleanup temp file if (fileBackedOutputStream != null) { try { fileBackedOutputStream.reset(); } catch (IOException e) { LOGGER.warn("Error occurred while deleting the temporary file. {}", e.getMessage()); } } } }
From source file:ddf.camel.component.content.ContentProducer.java
@Override public void process(Exchange exchange) throws ContentComponentException, ContentFrameworkException { LOGGER.debug("ENTERING: process"); if (!exchange.getPattern().equals(ExchangePattern.InOnly)) { return;/*ww w . j ava2 s . c o m*/ } Message in = exchange.getIn(); Object body = in.getBody(); File ingestedFile = null; if (body instanceof GenericFile) { GenericFile<File> genericFile = (GenericFile<File>) body; ingestedFile = (File) genericFile.getFile(); } else { LOGGER.warn("Unable to cast message body to Camel GenericFile, so unable to process ingested file"); throw new ContentComponentException( "Unable to cast message body to Camel GenericFile, so unable to process ingested file"); } if (ingestedFile == null) { LOGGER.debug("EXITING: process - ingestedFile is NULL"); return; } String operation = in.getHeader(Request.OPERATION, String.class); if (StringUtils.isEmpty(operation)) { throw new ContentComponentException("Unable to process file " + ingestedFile.getName() + " - Must specify an operation of create, read, update, or delete"); } String directive = in.getHeader(Request.DIRECTIVE, String.class); if (StringUtils.isEmpty(directive)) { throw new ContentComponentException("Unable to process file " + ingestedFile.getName() + " - Must specify a directive of STORE, PROCESS, or STORE_PROCESS"); } String contentUri = (String) in.getHeader(Request.CONTENT_URI, ""); LOGGER.debug("operation = {}", operation); LOGGER.debug("directive = {}", directive); LOGGER.debug("contentUri = {}", contentUri); FileInputStream fis = null; FileBackedOutputStream fbos = null; try { fis = FileUtils.openInputStream(ingestedFile); } catch (IOException e) { throw new ContentComponentException("Unable to open file {}" + ingestedFile.getName()); } Request.Directive requestDirective = Request.Directive.valueOf(directive); String fileExtension = FilenameUtils.getExtension(ingestedFile.getAbsolutePath()); String mimeType = null; if (fileExtension != null) { MimeTypeMapper mimeTypeMapper = endpoint.getComponent().getMimeTypeMapper(); if (mimeTypeMapper != null) { try { // XML files are mapped to multiple mime types, e.g., CSW, XML Metacard, etc. // If processing a .xml file, then use a stream that allows multiple reads on it // i.e., FileBackedOutputStream (FBOS), since the MimeTypeMapper will need to introspect // the xml file for its root element namespace and later the stream will need to // be read to persist it to the content store and/or create a metacard for ingest. // The FBOS is not used for every ingested file because do not want the performance // hit of potentially reading in a 1 GB NITF file that does not need to be introspected // to determine the mime type. if (fileExtension.equals("xml")) { // FBOS reads file into byte array in memory up to this threshold, then it transitions // to writing to a file. fbos = new FileBackedOutputStream(DEFAULT_FILE_BACKED_OUTPUT_STREAM_THRESHOLD); IOUtils.copy(fis, fbos); // Using fbos.asByteSource().openStream() allows us to pass in a copy of the InputStream mimeType = mimeTypeMapper.guessMimeType(fbos.asByteSource().openStream(), fileExtension); } else { mimeType = mimeTypeMapper.getMimeTypeForFileExtension(fileExtension); } } catch (MimeTypeResolutionException | IOException e) { IOUtils.closeQuietly(fis); IOUtils.closeQuietly(fbos); throw new ContentComponentException(e); } } else { IOUtils.closeQuietly(fis); LOGGER.error("Did not find a MimeTypeMapper service"); throw new ContentComponentException( "Unable to find a mime type for the ingested file " + ingestedFile.getName()); } } try { LOGGER.debug("Preparing content item for mimeType = {}", mimeType); if (StringUtils.isNotEmpty(mimeType)) { ContentItem newItem; // If the FileBackedOutputStream (FBOS) was used, then must be processing an XML file and need to // get a fresh InputStream from the FBOS to use in the content item if (fbos != null) { newItem = new IncomingContentItem(fbos.asByteSource().openStream(), mimeType, ingestedFile.getName()); } else { // Otherwise, the InputStream created from the ingested file has never been read, // so can use it in the content item newItem = new IncomingContentItem(fis, mimeType, ingestedFile.getName()); } newItem.setUri(contentUri); LOGGER.debug("Creating content item."); CreateRequest createRequest = new CreateRequestImpl(newItem, null); CreateResponse createResponse = endpoint.getComponent().getContentFramework().create(createRequest, requestDirective); if (createResponse != null) { ContentItem contentItem = createResponse.getCreatedContentItem(); LOGGER.debug("content item created with id = {}", contentItem.getId()); LOGGER.debug(contentItem.toString()); } } else { LOGGER.debug("mimeType is NULL"); throw new ContentComponentException( "Unable to determine mime type for the file " + ingestedFile.getName()); } } catch (IOException e) { throw new ContentComponentException( "Unable to determine mime type for the file " + ingestedFile.getName()); } finally { IOUtils.closeQuietly(fis); if (fbos != null) { try { fbos.reset(); } catch (IOException e) { LOGGER.info("Unable to reset FileBackedOutputStream"); } } } LOGGER.debug("EXITING: process"); }
From source file:org.andrewhitchcock.duwamish.Partition.java
public void before() { try {//from w w w . ja v a 2s.c o m // Sort incoming messages FileBackedOutputStream messages = new FileBackedOutputStream(incomingMessageThreshold); MergeSorter .create(org.andrewhitchcock.duwamish.protos.Duwamish.Message.class, Comparators.newMessageComparator()) .sort(messages, new BufferedInputStream(incomingMessages.getSupplier().getInput())); incomingMessages = new FileBackedOutputStream(incomingMessageThreshold); // Open readers vertexReader = ProtocolBufferReader.newReader(org.andrewhitchcock.duwamish.protos.Duwamish.Vertex.class, FileUtil.newInputStream(vertexFile)); edgeReader = ProtocolBufferReader.newReader(org.andrewhitchcock.duwamish.protos.Duwamish.Edge.class, FileUtil.newInputStream(edgeFile)); messageReader = ProtocolBufferReader.newReader( org.andrewhitchcock.duwamish.protos.Duwamish.Message.class, new BufferedInputStream(messages.getSupplier().getInput())); // Open new vertex value writer vertexFileWriter = FileUtil.newOutputStream(newVertexFile); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.andrewhitchcock.duwamish.util.MergeSorter.java
private BufferedOutputStream getNextOutputStream() { FileBackedOutputStream temp = new FileBackedOutputStream(1024 * 1024); // 1 MB mergeQueue.addLast(temp);//from w ww . j a va 2s .co m return new BufferedOutputStream(temp); }