List of usage examples for org.apache.commons.io.input BoundedInputStream BoundedInputStream
public BoundedInputStream(InputStream in, long size)
BoundedInputStream that wraps the given input stream and limits it to a certain size. From source file:com.msopentech.odatajclient.engine.it.MediaEntityUpdateTestITCase.java
private void updateMediaEntity(final ODataPubFormat format, final String prefer, final String image, final int id) throws Exception { URIBuilder builder = client.getURIBuilder(testDefaultServiceRootURL).appendEntityTypeSegment("Car") .appendKeySegment(id).appendValueSegment(); // The sample service has an upload request size of 65k InputStream input = new BoundedInputStream(getClass().getResourceAsStream(image), 65000); final byte[] expected = new byte[65000]; IOUtils.read(input, expected, 0, expected.length); IOUtils.closeQuietly(input);/*from w w w .j a va 2 s.co m*/ input = new BoundedInputStream(getClass().getResourceAsStream(image), 65000); final ODataMediaEntityUpdateRequest updateReq = client.getStreamedRequestFactory() .getMediaEntityUpdateRequest(builder.build(), input); updateReq.setFormat(format); updateReq.setPrefer(prefer); final URI uri = client.getURIBuilder(testDefaultServiceRootURL).appendEntityTypeSegment("Car") .appendKeySegment(id).build(); final String etag = getETag(uri); if (StringUtils.isNotBlank(etag)) { updateReq.setIfMatch(etag); } final MediaEntityUpdateStreamManager streamManager = updateReq.execute(); final ODataMediaEntityUpdateResponse updateRes = streamManager.getResponse(); assertEquals(204, updateRes.getStatusCode()); builder = client.getURIBuilder(testDefaultServiceRootURL).appendEntityTypeSegment("Car") .appendKeySegment(id).appendValueSegment(); final ODataMediaRequest retrieveReq = client.getRetrieveRequestFactory().getMediaRequest(builder.build()); final ODataRetrieveResponse<InputStream> retrieveRes = retrieveReq.execute(); assertEquals(200, retrieveRes.getStatusCode()); final byte[] actual = new byte[65000]; IOUtils.read(retrieveRes.getBody(), actual, 0, actual.length); assertTrue(new EqualsBuilder().append(expected, actual).isEquals()); }
From source file:net.jotel.ws.client.WebSocketReaderThread.java
@Override public void run() { try {/* ww w. j a va 2 s . co m*/ int inFrameType = 0; while (active.get() && !interrupted()) { int b = read(inputStream); boolean finalFragment = (b & 0x80) == 0x80; int opcode = b & 0xF; if (!finalFragment) { if (inFrameType == 0) { inFrameType = opcode; } } else { inFrameType = 0; } b = read(inputStream); final boolean masked = (b & 0x80) == 0x80; // FIXME in the spec length might be 64 bit.. but this implementation won't handle so big amount of data int length = b & 0x7f; if (length > 125) { int bytesToRead = (length == 126) ? 2 : 8; length = 0; for (int i = 0; i < bytesToRead; i++) { b = read(inputStream); length = length << (i << 3) | (b & 0xFF); // (i * 8) == (i << 3) } } byte[] mask = null; if (masked) { mask = new byte[WebSocketClient.MASK_SIZE]; read(inputStream, mask); } byte[] payload = new byte[length]; MaskedInputStream maskedInputStream = new MaskedInputStream( new BoundedInputStream(inputStream, length), mask); read(maskedInputStream, payload); wsClient.onDataFrame(opcode, payload); } } catch (Exception ex) { if (active.compareAndSet(true, false)) { wsClient.triggerOnError(ex); log.error(ex.getMessage(), ex); if (wsClient.isReconnectEnabled()) { new WebSocketReconnectionThread(wsClient).start(); } } } }
From source file:de.l3s.archivepig.enrich.Response.java
@Override public void enrich(Tuple data, Tuple enrichment, Object... params) throws Exception { long size = get(data, "_record.size"); long offset = get(data, "_record.offset"); String filename = get(data, "_record.filename"); String cdxFile = get(data, "_record.cdxFile"); if (size < 0 || offset < 0) return;//w w w . ja v a 2s . c o m FileSystem fs = FileSystem.get(UDFContext.getUDFContext().getJobConf()); Deque<String> cdxSegments = new ArrayDeque<String>(Lists.reverse(list(cdxFile.split("\\/")))); cdxSegments.pop(); // remove filename String pathExtension = ""; Path path = new Path(ArchiveLoader.dataPath(), pathExtension + filename); while (!fs.exists(path)) { if (cdxSegments.isEmpty()) { enrichment.append(new HashMap<String, String>()); enrichment.append(new HashMap<String, String>()); enrichment.append(null); return; } String cdxSegment = cdxSegments.pop(); if (cdxSegment.endsWith(".har")) cdxSegment = cdxSegment.substring(0, cdxSegment.length() - 4); pathExtension = cdxSegment + "/" + pathExtension; path = new Path(ArchiveLoader.dataPath(), pathExtension + filename); } FSDataInputStream fsin = fs.open(path); fsin.seek(offset); InputStream in = fsin; ByteArrayOutputStream recordOutput = new ByteArrayOutputStream(); try { try (BoundedInputStream boundedIn = new BoundedInputStream(in, size); ArchiveReader reader = ArchiveReaderFactory.get(filename, boundedIn, false);) { ArchiveRecord record; record = reader.get(); ArchiveRecordHeader header = record.getHeader(); enrichment.append(header.getHeaderFields()); record.dump(recordOutput); } catch (Exception e) { return; } finally { in.close(); recordOutput.close(); } } catch (Exception e) { return; } try (InputStream httpResponse = new ByteArrayInputStream(recordOutput.toByteArray())) { // ALL COMMENTS ARE NEW VERSION VARIANTS FOR HTTP-CORE 4.3, currently in use 4.2.5 // SessionInputBufferImpl sessionInputBuffer = new SessionInputBufferImpl(new HttpTransportMetricsImpl(), 2048); // sessionInputBuffer.bind(httpResponse); // DefaultHttpResponseParserFactory responseParserFactory = new DefaultHttpResponseParserFactory(); // HttpMessageParser<HttpResponse> responseParser = responseParserFactory.create(sessionInputBuffer, MessageConstraints.DEFAULT); // HttpResponse response = responseParser.parse(); // Header[] httpHeaders = response.getAllHeaders(); HttpResponseParser parser = new HttpResponseParser(); HttpResponse response = parser.parse(httpResponse); HttpHeaders httpHeaders = response.getHeaders(); Map<String, String> httpHeadersMap = new HashMap<String, String>(); for (HttpHeader httpHeader : httpHeaders) { httpHeadersMap.put(httpHeader.getName(), httpHeader.getValue()); } enrichment.append(httpHeadersMap); // byte[] payload = new byte[sessionInputBuffer.length()]; // sessionInputBuffer.read(payload); byte[] payload = IOUtils.toByteArray(response); enrichment.append(payload); // HttpEntity entity = new ByteArrayEntity(payload); // output.append(entity == null ? null : EntityUtils.toString(entity)); } catch (Exception ignored) { } }
From source file:edu.usc.irds.sparkler.plugin.HtmlUnitFetcher.java
@Override public FetchedData fetch(Resource resource) throws Exception { LOG.info("HtmlUnit FETCHER {}", resource.getUrl()); FetchedData fetchedData;/* w w w. j a v a 2 s . c o m*/ try { String userAgent = getUserAgent(); if (StringUtils.isNotBlank(userAgent)) { driver.removeRequestHeader(USER_AGENT); driver.addRequestHeader(USER_AGENT, userAgent); } Page page = driver.getPage(resource.getUrl()); WebResponse response = page.getWebResponse(); boolean truncated = false; try (InputStream stream = response.getContentAsStream()) { try (BoundedInputStream boundedStream = new BoundedInputStream(stream, CONTENT_LIMIT)) { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { IOUtils.copy(boundedStream, out); fetchedData = new FetchedData(out.toByteArray(), response.getContentType(), response.getStatusCode()); long contentLength = page.getWebResponse().getContentLength(); if (contentLength > 0 && contentLength < Integer.MAX_VALUE) { fetchedData.setContentLength((int) contentLength); truncated = (contentLength > fetchedData.getContentLength()); if (truncated) { LOG.info("Content Truncated: {}, TotalSize={}", resource.getUrl(), contentLength); } } } } } resource.setStatus(ResourceStatus.FETCHED.toString()); List<NameValuePair> respHeaders = page.getWebResponse().getResponseHeaders(); Map<String, List<String>> headers = new HashMap<>(); fetchedData.setHeaders(headers); if (respHeaders != null && !respHeaders.isEmpty()) { respHeaders.forEach(item -> { if (!headers.containsKey(item.getName())) { headers.put(item.getName(), new ArrayList<>()); } headers.get(item.getName()).add(item.getValue()); }); } if (truncated) { //add truncated header headers.put(TRUNCATED, Collections.singletonList(Boolean.TRUE.toString())); } } catch (Exception e) { LOG.warn(e.getMessage(), e); fetchedData = new FetchedData(new byte[0], "unknown/unknown", 0); // fixme: use proper status code resource.setStatus(ResourceStatus.ERROR.toString()); } fetchedData.setResource(resource); return fetchedData; }
From source file:com.eucalyptus.tokens.oidc.OidcDiscoveryCache.java
protected OidcResource resolve(final HttpURLConnection conn) throws IOException { SslSetup.configureHttpsUrlConnection(conn); try (final InputStream istr = conn.getInputStream()) { final int statusCode = conn.getResponseCode(); if (statusCode == 304) { return new OidcResource(statusCode); } else {/*from w ww . j av a2 s .co m*/ Certificate[] certs = new Certificate[0]; if (conn instanceof HttpsURLConnection) { certs = ((HttpsURLConnection) conn).getServerCertificates(); } final long contentLength = conn.getContentLengthLong(); if (contentLength > MAX_LENGTH) { throw new IOException(conn.getURL() + " content exceeds maximum size, " + MAX_LENGTH); } final byte[] content = ByteStreams.toByteArray(new BoundedInputStream(istr, MAX_LENGTH + 1)); if (content.length > MAX_LENGTH) { throw new IOException(conn.getURL() + " content exceeds maximum size, " + MAX_LENGTH); } return new OidcResource(statusCode, conn.getHeaderField(HttpHeaders.LAST_MODIFIED), conn.getHeaderField(HttpHeaders.ETAG), certs, content); } } }
From source file:io.hops.hopsworks.common.jobs.yarn.LogReader.java
/** * Writes all logs for a single container to the provided writer. * * @param valueStream//from w w w . j a v a2 s. c o m * @param writer * @throws IOException */ public static void readAcontainerLogs(DataInputStream valueStream, Writer writer) throws IOException { int bufferSize = 65536; char[] cbuf = new char[bufferSize]; String fileType; String fileLengthStr; long fileLength; while (true) { try { fileType = valueStream.readUTF(); } catch (EOFException e) { // EndOfFile return; } fileLengthStr = valueStream.readUTF(); fileLength = Long.parseLong(fileLengthStr); writer.write("\n\nLogType:"); writer.write(fileType); writer.write("\nLogLength:"); writer.write(fileLengthStr); writer.write("\nLog Contents:\n"); // ByteLevel BoundedInputStream bis = new BoundedInputStream(valueStream, fileLength); InputStreamReader reader = new InputStreamReader(bis); int currentRead = 0; int totalRead = 0; while ((currentRead = reader.read(cbuf, 0, bufferSize)) != -1) { writer.write(cbuf, 0, currentRead); totalRead += currentRead; } } }
From source file:edu.utsa.sifter.FileInfo.java
public Document generateDoc(final AbstractParser tika, final Analyzer analyzer) throws IOException { // System.out.println("generateDoc on " + fullPath()); String ext = null;//from ww w .ja va 2s . c o m try { ext = extension(); } catch (Exception ex) { System.err.println("failed adding extension"); ex.printStackTrace(System.err); } // wrap data in BoundedInputStream to only expose contents to tika return makeDoc(tika, analyzer, ID, new BoundedInputStream(Data, FileSize), Metadata, ext, fullPath().startsWith("$Unallocated/")); }
From source file:com.github.horrorho.inflatabledonkey.chunk.engine.ChunkListDecrypter.java
BoundedInputStream boundedInputStream(InputStream inputStream, int length) { BoundedInputStream bis = new BoundedInputStream(inputStream, length); // No close() required/ not propagated. bis.setPropagateClose(false);/* w w w . j av a 2 s . c o m*/ return bis; }
From source file:com.msopentech.odatajclient.engine.it.CreateMediaEntityTestITCase.java
private void createMediaEntity(final ODataPubFormat format, final String contentType, final String prefer, final String file) throws IOException { try {//from w ww . jav a 2s .c o m final URIBuilder uriBuilder = client.getURIBuilder(testDefaultServiceRootURL) .appendEntitySetSegment("Car"); // The sample service has an upload request size of 65k final InputStream inputStream = new BoundedInputStream(getClass().getResourceAsStream(file), 65000); final ODataMediaEntityCreateRequest createReq = client.getStreamedRequestFactory() .getMediaEntityCreateRequest(uriBuilder.build(), inputStream); createReq.setFormat(format); createReq.setContentType(contentType); createReq.setPrefer(prefer); final MediaEntityCreateStreamManager streamManager = createReq.execute(); final ODataMediaEntityCreateResponse createRes = streamManager.getResponse(); if (prefer.equals("return-content")) { assertEquals(201, createRes.getStatusCode()); final ODataEntity createdEntity = createRes.getBody(); assertNotNull(createdEntity); assertEquals(2, createdEntity.getProperties().size()); // get the vin property of the entity created final int id = "VIN".equals(createdEntity.getProperties().get(0).getName()) ? createdEntity.getProperties().get(0).getPrimitiveValue().<Integer>toCastValue() : createdEntity.getProperties().get(1).getPrimitiveValue().<Integer>toCastValue(); uriBuilder.appendKeySegment(id).appendValueSegment(); // get the stream value that got created final ODataMediaRequest retrieveReq = client.getRetrieveRequestFactory() .getMediaRequest(uriBuilder.build()); retrieveReq.setFormat(ODataMediaFormat.WILDCARD); final ODataRetrieveResponse<InputStream> retrieveRes = retrieveReq.execute(); assertEquals(200, retrieveRes.getStatusCode()); assertNotNull(retrieveRes.getBody()); String etag = retrieveRes.getEtag(); // get the entity created final ODataEntity entity = client.getObjectFactory() .newEntity("Microsoft.Test.OData.Services.AstoriaDefaultService.Car"); final ODataEntityRequest req = client.getRetrieveRequestFactory() .getEntityRequest(client.getURIBuilder(testDefaultServiceRootURL) .appendEntityTypeSegment("Car").appendKeySegment(id).build()); req.setFormat(format); final ODataRetrieveResponse<ODataEntity> res = req.execute(); final ODataEntity entityToBeUpdated = res.getBody(); assertNotNull(entity); String propertyName = "Description"; final String newValue = "New renault car - " + propertyName; ODataProperty propertyValue = entityToBeUpdated.getProperty(propertyName); if (propertyValue != null) { entityToBeUpdated.removeProperty(propertyValue); } // add new value for the property entityToBeUpdated.addProperty(client.getObjectFactory().newPrimitiveProperty(propertyName, client.getPrimitiveValueBuilder().setText(newValue).setType(EdmSimpleType.String).build())); UpdateType type = UpdateType.REPLACE; // update the entity in the server ODataEntityUpdateRequest updateReq = client.getCUDRequestFactory().getEntityUpdateRequest(type, entityToBeUpdated); req.setFormat(format); if (StringUtils.isNotBlank(etag)) { req.setIfMatch(etag); } ODataEntityUpdateResponse updateRes = updateReq.execute(); assertEquals(204, updateRes.getStatusCode()); final ODataEntityRequest afterUpdateReq = client.getRetrieveRequestFactory() .getEntityRequest(client.getURIBuilder(testDefaultServiceRootURL) .appendEntityTypeSegment("Car").appendKeySegment(id).build()); req.setFormat(format); //assert whether the original value is equal to the actual value that got updated. final ODataRetrieveResponse<ODataEntity> afterUpdateRes = afterUpdateReq.execute(); final ODataEntity entityAfterUpdate = afterUpdateRes.getBody(); assertEquals(newValue, entityAfterUpdate.getProperty("Description").getValue().toString()); // delete the entity URIBuilder deleteUriBuilder = client.getURIBuilder(testDefaultServiceRootURL) .appendEntityTypeSegment("Car(" + id + ")"); ODataDeleteRequest deleteReq = client.getCUDRequestFactory() .getDeleteRequest(deleteUriBuilder.build()); deleteReq.setFormat(format); deleteReq.setContentType(contentType); ODataDeleteResponse deleteRes = deleteReq.execute(); assertEquals(204, deleteRes.getStatusCode()); } else { assertEquals(204, createRes.getStatusCode()); } } catch (Exception e) { if ((format.equals(ODataPubFormat.JSON) || (format.equals(ODataPubFormat.JSON_NO_METADATA)) && e.getMessage().equals("No edit link found"))) { assertTrue(true); } else { fail(e.getMessage()); } } catch (AssertionError e) { fail(e.getMessage()); } }
From source file:com.streamsets.datacollector.execution.preview.sync.SyncPreviewer.java
@Override public RawPreview getRawSource(int maxLength, MultivaluedMap<String, String> previewParams) throws PipelineRuntimeException, PipelineStoreException { changeState(PreviewStatus.RUNNING, null); int bytesToRead = configuration.get(MAX_SOURCE_PREVIEW_SIZE_KEY, MAX_SOURCE_PREVIEW_SIZE_DEFAULT); bytesToRead = Math.min(bytesToRead, maxLength); PipelineConfiguration pipelineConf = pipelineStore.load(name, rev); if (pipelineConf.getStages().isEmpty()) { throw new PipelineRuntimeException(ContainerError.CONTAINER_0159, name); }//from www.ja v a 2s. c om //find the source stage in the pipeline configuration StageDefinition sourceStageDef = getSourceStageDef(pipelineConf); RawSourcePreviewer rawSourcePreviewer = createRawSourcePreviewer(sourceStageDef, previewParams); RawPreview rawPreview; ClassLoader classLoader = sourceStageDef.getStageClassLoader(); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(classLoader); try (BoundedInputStream bIn = new BoundedInputStream(rawSourcePreviewer.preview(bytesToRead), bytesToRead)) { rawPreview = new RawPreviewImpl(IOUtils.toString(bIn), rawSourcePreviewer.getMimeType()); } catch (IOException ex) { throw new PipelineRuntimeException(PreviewError.PREVIEW_0003, ex.toString(), ex); } finally { Thread.currentThread().setContextClassLoader(contextClassLoader); } changeState(PreviewStatus.FINISHED, null); return rawPreview; }