List of usage examples for com.google.common.io FileBackedOutputStream FileBackedOutputStream
public FileBackedOutputStream(int fileThreshold)
From source file:io.appium.java_client.remote.NewAppiumSessionPayload.java
private NewAppiumSessionPayload(Reader source, boolean forceMobileJSONWP) throws IOException { this.forceMobileJSONWP = forceMobileJSONWP; // Dedicate up to 10% of all RAM or 20% of available RAM (whichever is smaller) to storing this // payload./*www .j a va2s .c o m*/ int threshold = (int) Math.min(Integer.MAX_VALUE, Math.min(Runtime.getRuntime().freeMemory() / 5, Runtime.getRuntime().maxMemory() / 10)); backingStore = new FileBackedOutputStream(threshold); try (Writer writer = new OutputStreamWriter(backingStore, UTF_8)) { CharStreams.copy(source, writer); } ImmutableSet.Builder<CapabilitiesFilter> adapters = ImmutableSet.builder(); ServiceLoader.load(CapabilitiesFilter.class).forEach(adapters::add); adapters.add(new ChromeFilter()).add(new EdgeFilter()).add(new FirefoxFilter()) .add(new InternetExplorerFilter()).add(new OperaFilter()).add(new SafariFilter()); this.adapters = adapters.build(); ImmutableSet.Builder<CapabilityTransform> transforms = ImmutableSet.builder(); ServiceLoader.load(CapabilityTransform.class).forEach(transforms::add); transforms.add(new ProxyTransform()).add(new StripAnyPlatform()).add(new W3CPlatformNameNormaliser()); this.transforms = transforms.build(); ImmutableSet.Builder<Dialect> dialects = ImmutableSet.builder(); if (getOss() != null) { dialects.add(Dialect.OSS); } if (getAlwaysMatch() != null || getFirstMatch() != null) { dialects.add(Dialect.W3C); } validate(); }
From source file:ddf.catalog.core.ftp.ftplets.FtpRequestHandler.java
/** * @param session The current {@link FtpSession} * @param request The current {@link FtpRequest} * @return {@link FtpletResult#SKIP} - signals successful ingest and to discontinue and further processing on the FTP request * @throws FtpException general exception for Ftplets * @throws IOException thrown when there is an error fetching data from client */// www.j ava2s . c o m @Override public FtpletResult onUploadStart(FtpSession session, FtpRequest request) throws FtpException, IOException { LOGGER.debug("Beginning FTP ingest of {}", request.getArgument()); Subject shiroSubject = (Subject) session.getAttribute(SUBJECT); if (shiroSubject != null) { FtpFile ftpFile = null; String fileName = request.getArgument(); try { ftpFile = session.getFileSystemView().getFile(fileName); } catch (FtpException e) { LOGGER.error("Failed to retrieve file from FTP session"); } if (ftpFile == null) { LOGGER.error("Sending FTP status code 501 to client - syntax errors in request parameters"); session.write(new DefaultFtpReply(FtpReply.REPLY_501_SYNTAX_ERROR_IN_PARAMETERS_OR_ARGUMENTS, STOR_REQUEST)); throw new FtpException("File to be transferred from client did not exist"); } DataConnectionFactory connFactory = session.getDataConnection(); if (connFactory instanceof IODataConnectionFactory) { InetAddress address = ((IODataConnectionFactory) connFactory).getInetAddress(); if (address == null) { session.write(new DefaultFtpReply(FtpReply.REPLY_503_BAD_SEQUENCE_OF_COMMANDS, "PORT or PASV must be issued first")); LOGGER.error("Sending FTP status code 503 to client - PORT or PASV must be issued before STOR"); throw new FtpException("FTP client address was null"); } } if (!ftpFile.isWritable()) { session.write(new DefaultFtpReply(FtpReply.REPLY_550_REQUESTED_ACTION_NOT_TAKEN, "Insufficient permissions")); LOGGER.error("Sending FTP status code 550 to client - insufficient permissions to write file."); throw new FtpException("Insufficient permissions to write file"); } session.write(new DefaultFtpReply(FtpReply.REPLY_150_FILE_STATUS_OKAY, STOR_REQUEST + " " + fileName)); LOGGER.debug("Replying to client with code 150 - file status okay"); try (FileBackedOutputStream outputStream = new FileBackedOutputStream(1000000); final AutoCloseable ac = outputStream::reset) { DataConnection dataConnection = connFactory.openConnection(); dataConnection.transferFromClient(session, outputStream); session.write( new DefaultFtpReply(FtpReply.REPLY_226_CLOSING_DATA_CONNECTION, "Closing data connection")); LOGGER.debug("Sending FTP status code 226 to client - closing data connection"); String fileExtension = FilenameUtils.getExtension(ftpFile.getAbsolutePath()); String mimeType = getMimeType(fileExtension, outputStream); ContentItem newItem = new ContentItemImpl(outputStream.asByteSource(), mimeType, fileName, null); CreateStorageRequest createRequest = new CreateStorageRequestImpl( Collections.singletonList(newItem), null); createRequest.getProperties().put(SecurityConstants.SECURITY_SUBJECT, shiroSubject); CreateResponse createResponse = catalogFramework.create(createRequest); if (createResponse != null) { List<Metacard> createdMetacards = createResponse.getCreatedMetacards(); for (Metacard metacard : createdMetacards) { LOGGER.info("Content item created with id = {}", metacard.getId()); } } else { throw new FtpException(); } } catch (SourceUnavailableException | IngestException e) { LOGGER.error("Failure to ingest file {}", fileName, e); throw new FtpException("Failure to ingest file " + fileName); } catch (FtpException fe) { LOGGER.error("Failure to create metacard for file {}", fileName); throw new FtpException("Failure to create metacard for file " + fileName); } catch (Exception e) { LOGGER.error("Error getting the output data stream from the FTP session"); throw new IOException("Error getting the output stream from FTP session"); } finally { session.getDataConnection().closeDataConnection(); } } return FtpletResult.SKIP; }
From source file:org.codice.ddf.spatial.geocoding.extract.GeoNamesFileExtractor.java
/** * Download a GeoNames .zip file from a remote location * * @param resource - the name of the zip file to download ( ex. AD ) * @param response - the response from the get request * @param inputStream - the InputStream from the web connection * @param progressCallback - the callback to receive updates about the progress, may be * null if you don't want any updates * @throws GeoNamesRemoteDownloadException when the connection could not be established or the * file could not be downloaded. *//*from w ww. ja v a 2s. c o m*/ private InputStream getInputStreamFromUrl(String resource, Response response, InputStream inputStream, ProgressCallback progressCallback) throws GeoNamesRemoteDownloadException { int responseCode = 0; try (FileBackedOutputStream fileOutputStream = new FileBackedOutputStream(BUFFER_SIZE);) { responseCode = response.getStatus(); int totalFileSize = response.getLength(); if (inputStream == null) { throw new GeoNamesRemoteDownloadException( "Unable to get input stream from " + url + ". Server responded with : " + responseCode); } double totalBytesRead = 0.0; int bytesRead = -1; byte[] buffer = new byte[BUFFER_SIZE]; while ((bytesRead = inputStream.read(buffer)) != -1) { fileOutputStream.write(buffer, 0, bytesRead); totalBytesRead += bytesRead; if (progressCallback != null) { progressCallback.updateProgress((int) ((totalBytesRead / totalFileSize) * 50)); } } if (progressCallback != null) { progressCallback.updateProgress(50); } ByteSource byteSource = fileOutputStream.asByteSource(); fileOutputStream.flush(); inputStream.close(); closeConnection(); return byteSource.openBufferedStream(); } catch (IOException e) { throw new GeoNamesRemoteDownloadException("Unable to download " + resource + " from " + url + ". Server responded with : " + responseCode, e); } }
From source file:ddf.content.plugin.cataloger.CatalogContentPlugin.java
private Metacard generateMetacard(ContentItem contentItem, MimeType mimeType, String uri, InputStream message) throws MetacardCreationException { LOGGER.trace("ENTERING: generateMetacard"); List<InputTransformer> listOfCandidates = mimeTypeToTransformerMapper.findMatches(InputTransformer.class, mimeType);//from ww w . ja v a2s. co m LOGGER.debug("List of matches for mimeType [ {} ]: {}", mimeType, listOfCandidates); Metacard generatedMetacard = null; try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(1000000)) { long size; try { size = IOUtils.copyLarge(message, fileBackedOutputStream); LOGGER.debug("Copied {} bytes of file in content framework", size); } catch (IOException e) { throw new MetacardCreationException("Could not copy bytes of content message.", e); } // Multiple InputTransformers may be found that match the mime type. // Need to try each InputTransformer until we find one that can successfully transform // the input stream's data into a metacard. Once an InputTransformer is found that // can create the metacard, then do not need to try any remaining InputTransformers. for (InputTransformer transformer : listOfCandidates) { try (InputStream inputStreamMessageCopy = fileBackedOutputStream.asByteSource().openStream()) { generatedMetacard = transformer.transform(inputStreamMessageCopy); if (generatedMetacard != null) { try { Subject subject = SecurityUtils.getSubject(); if (subject != null) { generatedMetacard.setAttribute(new AttributeImpl(Metacard.POINT_OF_CONTACT, SubjectUtils.getName(subject))); } } catch (IllegalStateException e) { LOGGER.debug("Unable to retrieve user from request.", e); } if (uri != null) { //Setting the non-transformer specific information not including creation and modification dates/times generatedMetacard.setAttribute(new AttributeImpl(Metacard.RESOURCE_URI, uri)); generatedMetacard .setAttribute(new AttributeImpl(Metacard.RESOURCE_SIZE, String.valueOf(size))); } else { LOGGER.debug("Metacard had a null uri"); } if (StringUtils.isBlank(generatedMetacard.getTitle())) { LOGGER.debug("Metacard title was blank. Setting title to filename."); generatedMetacard .setAttribute(new AttributeImpl(Metacard.TITLE, contentItem.getFilename())); } break; } } catch (IOException | CatalogTransformerException e) { LOGGER.debug("Transformer [" + transformer + "] could not create metacard.", e); } } if (generatedMetacard == null) { throw new MetacardCreationException( "Could not create metacard with mimeType " + mimeType + ". No valid transformers found."); } LOGGER.trace("EXITING: generateMetacard"); } catch (IOException e) { LOGGER.debug("Error encountered while using filed backed stream.", e); } return generatedMetacard; }
From source file:de.faustedition.reasoning.InscriptionPrecedenceResource.java
@Get("svg|html") public Representation svg() throws IOException, ExecutionException, InterruptedException { final ExecutorService executorService = Executors.newCachedThreadPool(); final Process tred = new ProcessBuilder(environment.getRequiredProperty("graphviz.tred.path")).start(); final Process dot = new ProcessBuilder(environment.getRequiredProperty("graphviz.dot.path"), "-Tsvg") .start();/*from w w w .j a va 2s . c o m*/ executorService.submit(new Callable<Void>() { @Override public Void call() throws Exception { InputStream dataStream = null; OutputStream tredStream = null; try { ByteStreams.copy( dataStream = new ByteArrayInputStream(asDot().getBytes(Charset.forName("UTF-8"))), tredStream = tred.getOutputStream()); } finally { Closeables.close(dataStream, false); Closeables.close(tredStream, false); } return null; } }); executorService.submit(new Callable<Void>() { @Override public Void call() throws Exception { InputStream tredStream = null; OutputStream dotStream = null; try { ByteStreams.copy(tredStream = tred.getInputStream(), dotStream = dot.getOutputStream()); } finally { Closeables.close(tredStream, false); Closeables.close(dotStream, false); } return null; } }); final Future<FileBackedOutputStream> dotFuture = executorService .submit(new Callable<FileBackedOutputStream>() { @Override public FileBackedOutputStream call() throws Exception { final FileBackedOutputStream buf = new FileBackedOutputStream(102400); InputStream dotStream = null; try { ByteStreams.copy(dotStream = dot.getInputStream(), buf); } finally { Closeables.close(dotStream, false); Closeables.close(buf, false); } return buf; } }); Preconditions.checkState(tred.waitFor() == 0); Preconditions.checkState(dot.waitFor() == 0); final FileBackedOutputStream resultBuf = dotFuture.get(); return new OutputRepresentation(MediaType.IMAGE_SVG) { @Override public void write(OutputStream outputStream) throws IOException { ByteStreams.copy(resultBuf.getSupplier(), outputStream); resultBuf.reset(); } }; }
From source file:ddf.catalog.source.opensearch.OpenSearchSource.java
@Override public SourceResponse query(QueryRequest queryRequest) throws UnsupportedQueryException { String methodName = "query"; LOGGER.trace(methodName);//from w w w . j a va2s . c om Serializable metacardId = queryRequest.getPropertyValue(Metacard.ID); SourceResponseImpl response = null; Subject subject = null; WebClient restWebClient = null; if (queryRequest.hasProperties()) { Object subjectObj = queryRequest.getProperties().get(SecurityConstants.SECURITY_SUBJECT); subject = (Subject) subjectObj; } restWebClient = factory.getWebClientForSubject(subject); Query query = queryRequest.getQuery(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Received query: " + query); } boolean canDoOpenSearch = setOpenSearchParameters(query, subject, restWebClient); if (canDoOpenSearch) { InputStream responseStream = performRequest(restWebClient); response = new SourceResponseImpl(queryRequest, new ArrayList<Result>()); if (responseStream != null) { response = processResponse(responseStream, queryRequest); } } else { if (StringUtils.isEmpty((String) metacardId)) { OpenSearchFilterVisitor visitor = new OpenSearchFilterVisitor(); query.accept(visitor, null); metacardId = visitor.getMetacardId(); } restWebClient = newRestClient(query, (String) metacardId, false, subject); if (restWebClient != null) { InputStream responseStream = performRequest(restWebClient); Metacard metacard = null; List<Result> resultQueue = new ArrayList<Result>(); try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(1000000)) { if (responseStream != null) { IOUtils.copyLarge(responseStream, fileBackedOutputStream); InputTransformer inputTransformer = null; try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) { inputTransformer = getInputTransformer(inputStream); } catch (IOException e) { LOGGER.debug("Problem with transformation.", e); } if (inputTransformer != null) { try (InputStream inputStream = fileBackedOutputStream.asByteSource().openStream()) { metacard = inputTransformer.transform(inputStream); } catch (IOException e) { LOGGER.debug("Problem with transformation.", e); } } } } catch (IOException | CatalogTransformerException e) { LOGGER.debug("Problem with transformation.", e); } if (metacard != null) { metacard.setSourceId(getId()); ResultImpl result = new ResultImpl(metacard); resultQueue.add(result); response = new SourceResponseImpl(queryRequest, resultQueue); response.setHits(resultQueue.size()); } } } LOGGER.trace(methodName); return response; }
From source file:ddf.content.endpoint.rest.ContentEndpoint.java
CreateInfo parseAttachment(Attachment contentPart) {
CreateInfo createInfo = new CreateInfo();
InputStream stream = null;//from w ww . j av a 2s.c o m
FileBackedOutputStream fbos = null;
String filename = null;
String contentType = null;
// Get the file contents as an InputStream and ensure the stream is positioned
// at the beginning
try {
stream = contentPart.getDataHandler().getInputStream();
if (stream != null && stream.available() == 0) {
stream.reset();
}
createInfo.setStream(stream);
} catch (IOException e) {
LOGGER.warn("IOException reading stream from file attachment in multipart body", e);
}
// Example Content-Type header:
// Content-Type: application/json;id=geojson
if (contentPart.getContentType() != null) {
contentType = contentPart.getContentType().toString();
}
filename = contentPart.getContentDisposition().getParameter(FILENAME_CONTENT_DISPOSITION_PARAMETER_NAME);
// Only interested in attachments for file uploads. Any others should be covered by
// the FormParam arguments.
// If the filename was not specified, then generate a default filename based on the
// specified content type.
if (StringUtils.isEmpty(filename)) {
LOGGER.debug("No filename parameter provided - generating default filename");
String fileExtension = DEFAULT_FILE_EXTENSION;
try {
fileExtension = mimeTypeMapper.getFileExtensionForMimeType(contentType); // DDF-2307
if (StringUtils.isEmpty(fileExtension)) {
fileExtension = DEFAULT_FILE_EXTENSION;
}
} catch (MimeTypeResolutionException e) {
LOGGER.debug("Exception getting file extension for contentType = {}", contentType);
}
filename = DEFAULT_FILE_NAME + "." + fileExtension; // DDF-2263
LOGGER.debug("No filename parameter provided - default to {}", filename);
} else {
filename = FilenameUtils.getName(filename);
// DDF-908: filename with extension was specified by the client. If the
// contentType is null or the browser default, try to refine the contentType
// by determining the mime type based on the filename's extension.
if (StringUtils.isEmpty(contentType) || REFINEABLE_MIME_TYPES.contains(contentType)) {
String fileExtension = FilenameUtils.getExtension(filename);
LOGGER.debug("fileExtension = {}, contentType before refinement = {}", fileExtension, contentType);
if (fileExtension.equals("xml")) {
// FBOS reads file into byte array in memory up to this threshold, then it transitions
// to writing to a file.
fbos = new FileBackedOutputStream(DEFAULT_FILE_BACKED_OUTPUT_STREAM_THRESHOLD);
try {
IOUtils.copy(stream, fbos);
// Using fbos.asByteSource().openStream() allows us to pass in a copy of the InputStream
contentType = mimeTypeMapper.guessMimeType(fbos.asByteSource().openStream(), fileExtension);
createInfo.setStream(fbos.asByteSource().openStream());
} catch (IOException | MimeTypeResolutionException e) {
LOGGER.debug("Unable to refine contentType {} based on filename extension {}", contentType,
fileExtension);
}
} else {
try {
contentType = mimeTypeMapper.getMimeTypeForFileExtension(fileExtension);
} catch (MimeTypeResolutionException e) {
LOGGER.debug("Unable to refine contentType {} based on filename extension {}", contentType,
fileExtension);
}
}
LOGGER.debug("Refined contentType = {}", contentType);
}
}
createInfo.setContentType(contentType);
createInfo.setFilename(filename);
return createInfo;
}
From source file:org.codice.ddf.spatial.geocoding.extract.GeoNamesFileExtractor.java
/** * Unzips a file and returns the output as a new InputStream * * @param resource - the name of the resource file to be unzipped * @param inputStream - the InputStream for the file to be unzipped * @return - the unzipped file as an InputStream * @throws GeoEntryExtractionException when the given file fails to be unzipped. *//*from w w w . j av a 2 s . co m*/ private InputStream unZipInputStream(String resource, InputStream inputStream) throws GeoEntryExtractionException { try (FileBackedOutputStream bufferedOutputStream = new FileBackedOutputStream(BUFFER_SIZE); ZipInputStream zipInputStream = new ZipInputStream(inputStream);) { ZipEntry zipEntry; while ((zipEntry = zipInputStream.getNextEntry()) != null) { // GeoNames <filename>.zip files will contain <filename>.txt and readme.txt if (!zipEntry.getName().equals("readme.txt")) { byte data[] = new byte[BUFFER_SIZE]; int bytesRead; while ((bytesRead = zipInputStream.read(data, 0, BUFFER_SIZE)) != -1) { bufferedOutputStream.write(data, 0, bytesRead); } ByteSource zipByteSource = bufferedOutputStream.asByteSource(); bufferedOutputStream.flush(); fileSize = zipByteSource.size(); return zipByteSource.openBufferedStream(); } } } catch (IOException e) { throw new GeoEntryExtractionException("Unable to unzip " + resource, e); } throw new GeoEntryExtractionException("Unable to unzip " + resource); }
From source file:org.codice.ddf.commands.catalog.DumpCommand.java
private void writeMetacardToZip(ZipOutputStream zipOutputStream, Metacard metacard) { InputStream inputStream = null; try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(BUFFER_SIZE); ObjectOutputStream objectOutputStream = new ObjectOutputStream(fileBackedOutputStream)) { ZipEntry zipEntry = new ZipEntry(METACARD_PATH + metacard.getId()); zipOutputStream.putNextEntry(zipEntry); objectOutputStream.writeObject(new MetacardImpl(metacard)); inputStream = fileBackedOutputStream.asByteSource().openStream(); IOUtils.copy(inputStream, zipOutputStream); } catch (IOException e) { LOGGER.debug("Failed to add metacard with id {}.", metacard.getId(), e); } finally {// w w w. jav a2 s . co m try { if (inputStream != null) { inputStream.close(); } } catch (IOException e) { LOGGER.debug("Failed to close input stream", e); } } }
From source file:org.codice.ddf.endpoints.rest.RESTEndpoint.java
private Metacard generateMetacard(MimeType mimeType, String id, InputStream message) throws MetacardCreationException { List<InputTransformer> listOfCandidates = mimeTypeToTransformerMapper.findMatches(InputTransformer.class, mimeType);/*from w ww .j a v a 2 s . c o m*/ LOGGER.trace("Entering generateMetacard."); LOGGER.debug("List of matches for mimeType [{}]: {}", mimeType, listOfCandidates); Metacard generatedMetacard = null; try (FileBackedOutputStream fileBackedOutputStream = new FileBackedOutputStream(1000000)) { try { if (null != message) { IOUtils.copy(message, fileBackedOutputStream); } else { throw new MetacardCreationException( "Could not copy bytes of content message. Message was NULL."); } } catch (IOException e) { throw new MetacardCreationException("Could not copy bytes of content message.", e); } Iterator<InputTransformer> it = listOfCandidates.iterator(); StringBuilder causeMessage = new StringBuilder("Could not create metacard with mimeType "); causeMessage.append(mimeType); causeMessage.append(". Reason: "); while (it.hasNext()) { InputTransformer transformer = it.next(); try (InputStream inputStreamMessageCopy = fileBackedOutputStream.asByteSource().openStream()) { generatedMetacard = transformer.transform(inputStreamMessageCopy); } catch (CatalogTransformerException | IOException e) { causeMessage.append(System.lineSeparator()); causeMessage.append(e.getMessage()); // The caught exception more than likely does not have the root cause message // that is needed to inform the caller as to why things have failed. Therefore // we need to iterate through the chain of cause exceptions and gather up // all of their message details. Throwable cause = e.getCause(); while (null != cause && cause != cause.getCause()) { causeMessage.append(System.lineSeparator()); causeMessage.append(cause.getMessage()); cause = cause.getCause(); } LOGGER.debug("Transformer [{}] could not create metacard.", transformer, e); } if (generatedMetacard != null) { break; } } if (generatedMetacard == null) { throw new MetacardCreationException(causeMessage.toString()); } if (id != null) { generatedMetacard.setAttribute(new AttributeImpl(Metacard.ID, id)); } else { LOGGER.debug("Metacard had a null id"); } } catch (IOException e) { throw new MetacardCreationException("Could not create metacard.", e); } return generatedMetacard; }