List of usage examples for java.nio.channels Channels newChannel
public static WritableByteChannel newChannel(OutputStream out)
From source file:org.geotools.data.shapefile.ShpFiles.java
/** * @patch//from w w w.ja va 2 s . c om * * If an user name or password was established then it uses them * to login */ public ReadableByteChannel getReadChannel(ShpFileType type, FileReader requestor) throws IOException { URL url = acquireRead(type, requestor); ReadableByteChannel channel = null; try { if (isLocal()) { File file = DataUtilities.urlToFile(url); RandomAccessFile raf = new RandomAccessFile(file, "r"); channel = new FileChannelDecorator(raf.getChannel(), this, url, requestor); } else { InputStream in; if (compressed) { InputStream compressedIs = es.juntadeandalucia.panelGestion.negocio.utiles.Utils .getInputStream(url, user, password); in = es.juntadeandalucia.panelGestion.negocio.utiles.Utils .getShapefileFromCompressed(compressedIs, type); } else { in = RemoteFileReader.getInputStream(url, user, password); } channel = new ReadableByteChannelDecorator(Channels.newChannel(in), this, url, requestor); } } catch (Throwable e) { unlockRead(url, requestor); if (e instanceof IOException) { throw (IOException) e; } else if (e instanceof RuntimeException) { throw (RuntimeException) e; } else if (e instanceof Error) { throw (Error) e; } else { throw new RuntimeException(e); } } return channel; }
From source file:org.apache.nifi.remote.util.SiteToSiteRestApiClient.java
public void openConnectionForSend(final String transactionUrl, final Peer peer) throws IOException { final CommunicationsSession commSession = peer.getCommunicationsSession(); final String flowFilesPath = transactionUrl + "/flow-files"; final HttpPost post = createPost(flowFilesPath); // Set uri so that it'll be used as transit uri. ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(post.getURI().toString()); post.setHeader("Content-Type", "application/octet-stream"); post.setHeader("Accept", "text/plain"); post.setHeader(HttpHeaders.PROTOCOL_VERSION, String.valueOf(transportProtocolVersionNegotiator.getVersion())); setHandshakeProperties(post);/*from w ww .j a va2s. c o m*/ final CountDownLatch initConnectionLatch = new CountDownLatch(1); final URI requestUri = post.getURI(); final PipedOutputStream outputStream = new PipedOutputStream(); final PipedInputStream inputStream = new PipedInputStream(outputStream, DATA_PACKET_CHANNEL_READ_BUFFER_SIZE); final ReadableByteChannel dataPacketChannel = Channels.newChannel(inputStream); final HttpAsyncRequestProducer asyncRequestProducer = new HttpAsyncRequestProducer() { private final ByteBuffer buffer = ByteBuffer.allocate(DATA_PACKET_CHANNEL_READ_BUFFER_SIZE); private int totalRead = 0; private int totalProduced = 0; private boolean requestHasBeenReset = false; @Override public HttpHost getTarget() { return URIUtils.extractHost(requestUri); } @Override public HttpRequest generateRequest() throws IOException, HttpException { // Pass the output stream so that Site-to-Site client thread can send // data packet through this connection. logger.debug("sending data to {} has started...", flowFilesPath); ((HttpOutput) commSession.getOutput()).setOutputStream(outputStream); initConnectionLatch.countDown(); final BasicHttpEntity entity = new BasicHttpEntity(); entity.setChunked(true); entity.setContentType("application/octet-stream"); post.setEntity(entity); return post; } private final AtomicBoolean bufferHasRemainingData = new AtomicBoolean(false); /** * If the proxy server requires authentication, the same POST request has to be sent again. * The first request will result 407, then the next one will be sent with auth headers and actual data. * This method produces a content only when it's need to be sent, to avoid producing the flow-file contents twice. * Whether we need to wait auth is determined heuristically by the previous POST request which creates transaction. * See {@link SiteToSiteRestApiClient#initiateTransactionForSend(HttpPost)} for further detail. */ @Override public void produceContent(final ContentEncoder encoder, final IOControl ioControl) throws IOException { if (shouldCheckProxyAuth() && proxyAuthRequiresResend.get() && !requestHasBeenReset) { logger.debug("Need authentication with proxy server. Postpone producing content."); encoder.complete(); return; } if (bufferHasRemainingData.get()) { // If there's remaining buffer last time, send it first. writeBuffer(encoder); if (bufferHasRemainingData.get()) { return; } } int read; // This read() blocks until data becomes available, // or corresponding outputStream is closed. if ((read = dataPacketChannel.read(buffer)) > -1) { logger.trace("Read {} bytes from dataPacketChannel. {}", read, flowFilesPath); totalRead += read; buffer.flip(); writeBuffer(encoder); } else { final long totalWritten = commSession.getOutput().getBytesWritten(); logger.debug( "sending data to {} has reached to its end. produced {} bytes by reading {} bytes from channel. {} bytes written in this transaction.", flowFilesPath, totalProduced, totalRead, totalWritten); if (totalRead != totalWritten || totalProduced != totalWritten) { final String msg = "Sending data to %s has reached to its end, but produced : read : wrote byte sizes (%d : %d : %d) were not equal. Something went wrong."; throw new RuntimeException( String.format(msg, flowFilesPath, totalProduced, totalRead, totalWritten)); } transferDataLatch.countDown(); encoder.complete(); dataPacketChannel.close(); } } private void writeBuffer(ContentEncoder encoder) throws IOException { while (buffer.hasRemaining()) { final int written = encoder.write(buffer); logger.trace("written {} bytes to encoder.", written); if (written == 0) { logger.trace("Buffer still has remaining. {}", buffer); bufferHasRemainingData.set(true); return; } totalProduced += written; } bufferHasRemainingData.set(false); buffer.clear(); } @Override public void requestCompleted(final HttpContext context) { logger.debug("Sending data to {} completed.", flowFilesPath); debugProxyAuthState(context); } @Override public void failed(final Exception ex) { final String msg = String.format("Failed to send data to %s due to %s", flowFilesPath, ex.toString()); logger.error(msg, ex); eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, msg); } @Override public boolean isRepeatable() { // In order to pass authentication, request has to be repeatable. return true; } @Override public void resetRequest() throws IOException { logger.debug("Sending data request to {} has been reset...", flowFilesPath); requestHasBeenReset = true; } @Override public void close() throws IOException { logger.debug("Closing sending data request to {}", flowFilesPath); closeSilently(outputStream); closeSilently(dataPacketChannel); stopExtendingTtl(); } }; postResult = getHttpAsyncClient().execute(asyncRequestProducer, new BasicAsyncResponseConsumer(), null); try { // Need to wait the post request actually started so that we can write to its output stream. if (!initConnectionLatch.await(connectTimeoutMillis, TimeUnit.MILLISECONDS)) { throw new IOException("Awaiting initConnectionLatch has been timeout."); } // Started. transferDataLatch = new CountDownLatch(1); startExtendingTtl(transactionUrl, dataPacketChannel, null); } catch (final InterruptedException e) { throw new IOException("Awaiting initConnectionLatch has been interrupted.", e); } }
From source file:com.linkedin.databus.core.TestDbusEventBufferMult.java
private void batchReading(Set<Integer> srcIds, final int expectEvents, final int batchFetchSize) throws IOException, ScnNotFoundException, DatabusException, OffsetNotFoundException { // for debug generate this line String inputSrcIds = Arrays.toString(srcIds.toArray()); LOG.info("Reading events from " + inputSrcIds); // create checkpoints CheckpointMult cpMult = new CheckpointMult(); for (PhysicalSourceStaticConfig pConf : _pConfigs) { PhysicalPartition pPart = pConf.getPhysicalPartition(); Checkpoint cp;//from ww w . j a v a 2 s . co m if (cpMult.getCheckpoint(pPart) == null) { // needs a new checkpoint cp = new Checkpoint(); cp.setFlexible(); cpMult.addCheckpoint(pPart, cp); } } DbusEventBufferBatchReadable read = _eventBufferMult.getDbusEventBufferBatchReadable(srcIds, cpMult, null); int totalRead = 0; int numEventsRead = Integer.MAX_VALUE; int numNonControlEventsRead = 0; int maxIterNum = 100, iterCount = 0; String prevEvent = ""; while (numEventsRead > 0) { if (iterCount++ > maxIterNum) { fail("Checkpoint doesn't work - it is a never-ending loop"); } ByteArrayOutputStream jsonOut = new ByteArrayOutputStream(); WritableByteChannel jsonOutChannel = Channels.newChannel(jsonOut); numEventsRead = read.streamEvents(false, batchFetchSize, jsonOutChannel, Encoding.JSON_PLAIN_VALUE, new SourceDbusFilter(srcIds)).getNumEventsStreamed(); totalRead += numEventsRead; LOG.info("read for " + inputSrcIds + ": " + numEventsRead + " events"); byte[] jsonBytes = jsonOut.toByteArray(); if (jsonBytes.length == 0) break; // nothing more to read String jsonString = new String(jsonBytes); String[] jsonStrings = jsonString.split("\n"); assertEquals(jsonStrings.length, numEventsRead); ObjectMapper mapper = new ObjectMapper(); for (int i = 0; i < jsonStrings.length; i++) { // verify what was written String evtStr = jsonStrings[i]; if (evtStr.equals(prevEvent)) { // It may so happen that we receive the same event twice, especially when the // offered buffer is small. This check gets around the issue. continue; } prevEvent = evtStr; Map<String, Object> jsonMap = mapper.readValue(evtStr, new TypeReference<Map<String, Object>>() { }); //assertEquals(jsonMap.size(), 10); Integer srcId = (Integer) jsonMap.get("srcId"); if (!DbusEventUtils.isControlSrcId(srcId)) { // not a control message numNonControlEventsRead++; Integer physicalPartitionId = (Integer) jsonMap.get("physicalPartitionId"); Integer logicalPartitionId = (Integer) jsonMap.get("logicalPartitionId"); PhysicalPartition pPartition = _eventBufferMult.getPhysicalPartition(srcId, new LogicalPartition(logicalPartitionId.shortValue())); LOG.info("EVENT: " + jsonMap.toString()); assertTrue(srcIds.contains(srcId), "src id " + srcId + " doesn't match to " + inputSrcIds); assertEquals(physicalPartitionId, pPartition.getId(), "physical partition id didn't match"); } else { LOG.info("Control event: " + jsonMap.toString()); } } } assertTrue(totalRead > numNonControlEventsRead); assertEquals(numNonControlEventsRead, expectEvents); }
From source file:org.geotools.data.shapefile.ShpFiles.java
/** * Obtain a WritableByteChannel from the given URL. If the url protocol is file, a FileChannel * will be returned. Currently, this method will return a generic channel for remote urls, * however both shape and dbf writing can only occur with a local FileChannel channel. * /*from ww w.j a va2s . co m*/ * <p> * A write lock is obtained when this method is called and released when the channel is closed. * </p> * * * @param type * the type of file to open the stream to. * @param requestor * the object requesting the stream * * @return a WritableByteChannel for the provided file type * * @throws IOException * if there is an error opening the stream */ public WritableByteChannel getWriteChannel(ShpFileType type, FileWriter requestor) throws IOException { URL url = acquireWrite(type, requestor); try { WritableByteChannel channel; if (isLocal()) { File file = DataUtilities.urlToFile(url); RandomAccessFile raf = new RandomAccessFile(file, "rw"); channel = new FileChannelDecorator(raf.getChannel(), this, url, requestor); ((FileChannel) channel).lock(); } else { OutputStream out = url.openConnection().getOutputStream(); channel = new WritableByteChannelDecorator(Channels.newChannel(out), this, url, requestor); } return channel; } catch (Throwable e) { unlockWrite(url, requestor); if (e instanceof IOException) { throw (IOException) e; } else if (e instanceof RuntimeException) { throw (RuntimeException) e; } else if (e instanceof Error) { throw (Error) e; } else { throw new RuntimeException(e); } } }
From source file:org.commoncrawl.hadoop.io.deprecated.ArcFileReader.java
@Test public void testReader(File file) throws Exception { checkCRLFStateMachine();//from w ww .ja va 2 s . co m setIOTimeoutValue(30000); resetState(); Thread thread = new Thread(new Runnable() { public void run() { try { while (hasMoreItems()) { ArcFileItem item = new ArcFileItem(); getNextItem(item); LOG.info("GOT Item URL:" + item.getUri() + " StreamPos:" + item.getArcFilePos() + " Content Length:" + item.getContent().getCount()); for (ArcFileHeaderItem headerItem : item.getHeaderItems()) { if (headerItem.isFieldDirty(ArcFileHeaderItem.Field_ITEMKEY)) { // LOG.info("Header Item:" + headerItem.getItemKey() + " :" + // headerItem.getItemValue()); } else { // LOG.info("Header Item:" + headerItem.getItemValue()); } } // LOG.info("Content Length:" + item.getContent().getCount()); // LOG.info("Content:"); /* * ByteArrayInputStream inputStream = new * ByteArrayInputStream(item.getContent * ().getReadOnlyBytes(),0,item.getContent().getCount()); * BufferedReader reader = new BufferedReader(new * InputStreamReader(inputStream,Charset.forName("ASCII"))); String * line = null; while ((line = reader.readLine()) != null) { * LOG.info(line); } */ } LOG.info("NO MORE ITEMS... BYE"); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } } }); // run the thread ... thread.start(); ReadableByteChannel channel = Channels.newChannel(new FileInputStream(file)); try { int totalBytesRead = 0; for (;;) { ByteBuffer buffer = ByteBuffer.allocate(32768); int bytesRead = channel.read(buffer); // LOG.info("Read "+bytesRead + " From File"); if (bytesRead == -1) { finished(); break; } else { buffer.flip(); totalBytesRead += buffer.remaining(); available(buffer); } } } finally { channel.close(); } // now wait for thread to die ... LOG.info("Done Reading File.... Waiting for ArcFileThread to DIE"); thread.join(); LOG.info("Done Reading File.... ArcFileThread to DIED"); }
From source file:org.commoncrawl.util.ArcFileReader.java
@Test public void testReader(InputStream stream) throws IOException { setIOTimeoutValue(30000);/*from ww w . java2 s. co m*/ resetState(); Thread thread = new Thread(new Runnable() { public void run() { try { while (hasMoreItems()) { ArcFileItem item = new ArcFileItem(); getNextItem(item); LOG.info("GOT Item URL:" + item.getUri() + " StreamPos:" + item.getArcFilePos() + " Content Length:" + item.getContent().getCount()); for (ArcFileHeaderItem headerItem : item.getHeaderItems()) { if (headerItem.isFieldDirty(ArcFileHeaderItem.Field_ITEMKEY)) { // LOG.info("Header Item:" + headerItem.getItemKey() + " :" + // headerItem.getItemValue()); } else { // LOG.info("Header Item:" + headerItem.getItemValue()); } } // LOG.info("Content Length:" + item.getContent().getCount()); // LOG.info("Content:"); /* * ByteArrayInputStream inputStream = new * ByteArrayInputStream(item.getContent * ().getReadOnlyBytes(),0,item.getContent().getCount()); * BufferedReader reader = new BufferedReader(new * InputStreamReader(inputStream,Charset.forName("ASCII"))); String * line = null; while ((line = reader.readLine()) != null) { * LOG.info(line); } */ } LOG.info("NO MORE ITEMS... BYE"); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } } }); // run the thread ... thread.start(); ReadableByteChannel channel = Channels.newChannel(stream); try { int totalBytesRead = 0; for (;;) { ByteBuffer buffer = ByteBuffer.allocate(32768); int bytesRead = channel.read(buffer); // LOG.info("Read "+bytesRead + " From File"); if (bytesRead == -1) { finished(); break; } else { buffer.flip(); totalBytesRead += buffer.remaining(); available(buffer); } } } finally { channel.close(); } // now wait for thread to die ... LOG.info("Done Reading File.... Waiting for ArcFileThread to DIE"); try { thread.join(); } catch (InterruptedException e) { e.printStackTrace(); } LOG.info("Done Reading File.... ArcFileThread to DIED"); }
From source file:model.settings.ReadSettings.java
/** * Update the entire program./*from w w w . j a v a2 s . com*/ */ public static void update(final View _view, final boolean _showNoUpdateMSG) { new Thread() { public void run() { final boolean newReleaseAccepted = checkForUpdate(_view, _showNoUpdateMSG); if (newReleaseAccepted) { _view.dispose(); /* * dispose current version of paint that is currently running. */ InformationWindow iw = new InformationWindow("Update current version of Paint"); iw.setText("\t\t\tPaint update\n" + "This window will give information on the\n" + "update process of the paint - program.\n\n" + "The Program is directly pulled from the github\n" + "update page which can be found at\n" + "https://github.com/juliusHuelsmann/paint.git.\n" + "\nThe update-performance depends on your\n" + " internet connection.\n\n"); /* * Property name for getting operating system identifier. */ final String os_propertyName = "os.name"; /* * Different property content values for deciding which * operating system is used. */ final String propertyLinux = "Linux"; final String propertyOSX = "Mac OS X"; final String propertyWindows = "Windows"; /* * The identifier for the currently used operating system. */ final String propertyContent = System.getProperties().getProperty(os_propertyName); /* * Compute TEMP directory path depending on the currently * used operating system. */ iw.appendNewOperation("Checking operating system:"); iw.startWaiting(true); final String temp; if (propertyContent.equals(propertyLinux)) { temp = System.getenv().get("TMPDIR"); iw.appendNewResult("Linux"); } else if (propertyContent.equals(propertyWindows)) { temp = System.getenv().get("TMPDIR"); iw.appendNewResult("Windows"); } else if (propertyContent.equals(propertyOSX)) { temp = System.getenv().get("TMPDIR"); iw.appendNewResult("OS X"); } else { temp = System.getenv().get("TMPDIR"); iw.appendNewResult("Not found, System not implemented " + "yet. \nThis may cause errors."); throw new UnsupportedOperationException("not impl. yet."); } /* * Create sub-TEMP directory */ final String tempDirPaint = temp + "paint/"; final String ret0_a, command0 = "mkdir " + tempDirPaint; iw.appendNewOperation("Check whether sub-temp dir exists"); if (new File(tempDirPaint).exists()) { ret0_a = "The file already exists: "; // + tempDirPaint + "." ; iw.appendNewResult(ret0_a); // remove file iw.appendNewOperation("Remove old temp file."); final String ret0_b, command0_b = "rm -r -f " + tempDirPaint; ret0_b = Util.executeCommandLinux(command0_b); iw.appendNewResult(ret0_b); } iw.appendNewOperation("Create sub-TEMP directory"); final String ret0 = Util.executeCommandLinux(command0); iw.appendNewResult(ret0); /* * Check whether git is installted at the machine. */ iw.appendNewOperation("Clone Project into TEMP directory."); iw.appendNewOperation("Check whether git is installed."); final String command1 = "git version"; String ret1 = Util.executeCommandLinux(command1); boolean installed = ret1.contains(Util.EXECUTION_SUCCESS); if (installed) { iw.appendNewResult("Git installed."); /* * Clone project into TEMP directory */ iw.appendNewOperation("Clone Project into TEMP directory."); final String command1a = "git clone " + "https://github.com/juliusHuelsmann/paint.git " + tempDirPaint; String ret1a = Util.executeCommandLinux(command1a); iw.appendNewResult(ret1a); /* * Start jar file which copies itself into the program directory. */ iw.appendNewOperation("Launching new project file"); final String command2a = "java -jar " + tempDirPaint + "PaintNotes/paint.jar"; String ret2a = Util.executeCommandLinux(command2a); iw.appendNewResult(ret2a); } else { iw.appendNewOperation("\t Git not installed. Manual download."); /* * Download Program from repository URL */ final String repoURL = "https://github.com/juliusHuelsmann/paint/archive/master.zip"; final String zipPath = tempDirPaint + "master.zip"; try { iw.appendNewResult("Download zip"); URL website = new URL(repoURL); ReadableByteChannel rbc = Channels.newChannel(website.openStream()); FileOutputStream fos; fos = new FileOutputStream(zipPath); fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); fos.close(); } catch (FileNotFoundException e) { e.printStackTrace(); iw.appendNewResult("Failed download. exit."); try { Thread.sleep(2000); } catch (InterruptedException e1) { e1.printStackTrace(); } System.exit(1); } catch (IOException e) { e.printStackTrace(); iw.appendNewResult("Failed download. exit."); try { Thread.sleep(2000); } catch (InterruptedException e1) { e1.printStackTrace(); } System.exit(1); } /* * Unzip the program */ iw.appendNewOperation("unzip"); /** * The command which is executed for unzipping the program. */ final String commandUnzip = "unzip " + zipPath + " -d " + tempDirPaint; /** * The result of the command's execution in terminal. * If the response tells that the command has been executed * successfully, there is nothing to do. Otherwise * perform rotation done by program and print a warning. */ final String resultUnzip = Util.executeCommandLinux(commandUnzip); iw.appendNewResult(resultUnzip); if (resultUnzip.startsWith(Util.EXECUTION_SUCCESS)) { //print success information model.settings.State.getLogger().info("Download and execution successfull"); } else if (resultUnzip.startsWith(Util.EXECUTION_FAILED)) { // model.settings.State.getLogger().severe("Download and execution failed" + resultUnzip); try { Thread.sleep(2000); } catch (InterruptedException e1) { e1.printStackTrace(); } System.exit(1); } // /** * Command for removing zip download file. */ iw.appendNewOperation("Remove zip file"); final String commandMv = "rm " + zipPath; final String resultClear = Util.executeCommandLinux(commandMv); iw.appendNewResult(resultClear); /* * Start jar file which copies itself into the program directory. */ iw.appendNewOperation("Launching new project file"); final String command2a = "java -jar " + tempDirPaint + "paint-master/PaintNotes/paint.jar"; String ret2a = Util.executeCommandLinux(command2a); iw.appendNewResult(ret2a); } iw.stopWaiting(); try { Thread.sleep(2000); } catch (InterruptedException e1) { e1.printStackTrace(); } System.exit(1); // // // /* // * Check for version number // */ // final String command1 = "git clone " // + "https://github.com/juliusHuelsmann/paint.git " // + tempDirPaint; // String ret2 = Util.executeCommandLinux(command1); // model.settings.State.getLogger().severe("Executed" // + "\nC1:\t" + command0 + "\n\t" + ret0_a // + "\nC2:\t" + command1 + "\n\t" + ret1 // + "\nC3:\t" + command2 + "\n\t" + ret2); } } }.start(); }
From source file:org.jnetstream.capture.Captures.java
/** * Determines the format type of the supplied "input". Checks the format of * the supplied input and expects it to be in one of the supported format * types such as Pcap, etc../*from w w w .j a v a 2s.c o m*/ * * @param in * input to check and return format type * @return format type of the supplied input or null if format type is unknown * or not supported */ public static FormatType formatType(final InputStream in) throws IOException { return Captures.getLocal().formatType(Channels.newChannel(in)); }
From source file:org.forgerock.openidm.maintenance.upgrade.UpdateManagerImpl.java
/** * Fetch the zip file from the URL and write it to the local filesystem. * * @param url/* w w w. j a v a2 s . c om*/ * @return * @throws UpdateException */ private Path readZipFile(URL url) throws UpdateException { // Download the patch file final ReadableByteChannel channel; try { channel = Channels.newChannel(url.openStream()); } catch (IOException ex) { throw new UpdateException("Failed to access the specified file " + url + " " + ex.getMessage(), ex); } String workingDir = ""; final String targetFileName = new File(url.getPath()).getName(); final File patchDir = ARCHIVE_PATH.toFile(); patchDir.mkdirs(); final File targetFile = new File(patchDir, targetFileName); final FileOutputStream fos; try { fos = new FileOutputStream(targetFile); } catch (FileNotFoundException ex) { throw new UpdateException("Error in getting the specified file to " + targetFile, ex); } try { fos.getChannel().transferFrom(channel, 0, Long.MAX_VALUE); System.out.println("Downloaded to " + targetFile); } catch (IOException ex) { throw new UpdateException("Failed to get the specified file " + url + " to: " + targetFile, ex); } return targetFile.toPath(); }