List of usage examples for org.apache.hadoop.util Progressable progress
public void progress();
From source file:com.asakusafw.runtime.compatibility.hadoop1.JobCompatibilityHadoop1.java
License:Apache License
@Override public TaskAttemptContext newTaskAttemptContext(Configuration conf, TaskAttemptID id, final Progressable progressable) { if (conf == null) { throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$ }/*from w ww . j a va2 s . co m*/ if (id == null) { throw new IllegalArgumentException("id must not be null"); //$NON-NLS-1$ } if (progressable == null) { return new TaskAttemptContext(conf, id); } return new TaskAttemptContext(conf, id) { @Override public void progress() { progressable.progress(); super.progress(); } }; }
From source file:com.asakusafw.runtime.compatibility.hadoop2.JobCompatibilityHadoop2.java
License:Apache License
@Override public TaskAttemptContext newTaskAttemptContext(Configuration conf, TaskAttemptID id, final Progressable progressable) { if (conf == null) { throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$ }//from w w w. j a va 2 s. com if (id == null) { throw new IllegalArgumentException("id must not be null"); //$NON-NLS-1$ } if (progressable == null) { return new TaskAttemptContextImpl(conf, id); } return new TaskAttemptContextImpl(conf, id) { @Override public void progress() { progressable.progress(); super.progress(); } }; }
From source file:com.cloudera.oryx.als.computation.ComputationDataUtils.java
License:Open Source License
private static LongSet readExpectedIDs(String key, Progressable progressable, Configuration conf) throws IOException { LongSet ids = new LongSet(); long count = 0; for (long id : new AvroFileSource<Long>(Namespaces.toPath(key), Avros.longs()).read(conf)) { ids.add(id);//from ww w . j a v a2 s . co m if (++count % 10000 == 0) { progressable.progress(); } } log.info("Read {} IDs from {}", ids.size(), key); return ids; }
From source file:com.ricemap.spateDB.core.GridRecordWriter.java
License:Apache License
/** * Close the whole writer. Finalize all cell files and concatenate them * into the output file./* w w w .jav a2 s . c om*/ */ public synchronized void close(Progressable progressable) throws IOException { // Close all output files for (int cellIndex = 0; cellIndex < intermediateCellStreams.length; cellIndex++) { if (intermediateCellStreams[cellIndex] != null) { closeCell(cellIndex); } // Indicate progress. Useful if closing a single cell takes a long time if (progressable != null) progressable.progress(); } while (!closingThreads.isEmpty()) { try { Thread t = closingThreads.get(0); switch (t.getState()) { case NEW: t.start(); break; case TERMINATED: closingThreads.remove(0); break; default: t.join(10000); } } catch (InterruptedException e) { e.printStackTrace(); } } if (masterFile != null) masterFile.close(); }
From source file:edu.umn.cs.spatialHadoop.core.GridRecordWriter.java
License:Open Source License
/** * Close the whole writer. Finalize all cell files and concatenate them * into the output file.//from ww w . j a v a 2s . c o m */ public synchronized void close(Progressable progressable) throws IOException { // Close all output files for (int cellIndex = 0; cellIndex < intermediateCellStreams.length; cellIndex++) { if (intermediateCellStreams[cellIndex] != null) { closeCell(cellIndex); } // Indicate progress. Useful if closing a single cell takes a long time if (progressable != null) progressable.progress(); } LOG.info("Closing record writer with " + closingThreads.size() + " remaining threads"); while (!closingThreads.isEmpty()) { try { Thread t = closingThreads.get(0); switch (t.getState()) { case NEW: t.start(); break; case TERMINATED: closingThreads.remove(0); break; default: // Use limited time join to indicate progress frequently t.join(10000); } // Indicate progress. Useful if closing a single cell takes a long time if (progressable != null) progressable.progress(); } catch (InterruptedException e) { e.printStackTrace(); } } if (masterFile != null) masterFile.close(); }
From source file:io.druid.indexer.JobHelper.java
License:Apache License
public static void writeSegmentDescriptor(final FileSystem outputFS, final DataSegment segment, final Path descriptorPath, final Progressable progressable) throws IOException { final DataPusher descriptorPusher = (DataPusher) RetryProxy.create(DataPusher.class, new DataPusher() { @Override/* ww w.j av a 2 s .c o m*/ public long push() throws IOException { try { progressable.progress(); if (outputFS.exists(descriptorPath)) { if (!outputFS.delete(descriptorPath, false)) { throw new IOException( String.format("Failed to delete descriptor at [%s]", descriptorPath)); } } try (final OutputStream descriptorOut = outputFS.create(descriptorPath, true, DEFAULT_FS_BUFFER_SIZE, progressable)) { HadoopDruidIndexerConfig.jsonMapper.writeValue(descriptorOut, segment); descriptorOut.flush(); } } catch (RuntimeException | IOException ex) { log.info(ex, "Exception in descriptor pusher retry loop"); throw ex; } return -1; } }, RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS)); descriptorPusher.push(); }
From source file:io.druid.indexer.JobHelper.java
License:Apache License
public static long copyFileToZipStream(File file, ZipOutputStream zipOutputStream, Progressable progressable) throws IOException { createNewZipEntry(zipOutputStream, file); long numRead = 0; try (FileInputStream inputStream = new FileInputStream(file)) { byte[] buf = new byte[0x10000]; for (int bytesRead = inputStream.read(buf); bytesRead >= 0; bytesRead = inputStream.read(buf)) { progressable.progress(); if (bytesRead == 0) { continue; }/*from www . j a v a2 s. c o m*/ zipOutputStream.write(buf, 0, bytesRead); progressable.progress(); numRead += bytesRead; } } zipOutputStream.closeEntry(); progressable.progress(); return numRead; }
From source file:io.druid.indexer.JobHelper.java
License:Apache License
public static long unzipNoGuava(final Path zip, final Configuration configuration, final File outDir, final Progressable progressable) throws IOException { final DataPusher zipPusher = (DataPusher) RetryProxy.create(DataPusher.class, new DataPusher() { @Override/*from ww w .j a v a 2 s . co m*/ public long push() throws IOException { try { final FileSystem fileSystem = zip.getFileSystem(configuration); long size = 0L; final byte[] buffer = new byte[1 << 13]; progressable.progress(); try (ZipInputStream in = new ZipInputStream(fileSystem.open(zip, 1 << 13))) { for (ZipEntry entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) { final String fileName = entry.getName(); try (final OutputStream out = new BufferedOutputStream( new FileOutputStream(outDir.getAbsolutePath() + File.separator + fileName), 1 << 13)) { for (int len = in.read(buffer); len >= 0; len = in.read(buffer)) { progressable.progress(); if (len == 0) { continue; } size += len; out.write(buffer, 0, len); } out.flush(); } } } progressable.progress(); return size; } catch (IOException | RuntimeException exception) { log.error(exception, "Exception in unzip retry loop"); throw exception; } } }, RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS)); return zipPusher.push(); }
From source file:io.hops.erasure_coding.Encoder.java
License:Apache License
/** * Recovers a corrupt block in a parity file to an output stream. * <p/>//w w w . j av a2 s . com * The encoder generates codec.parityLength parity blocks for a source file * stripe. * Since there is only one output provided, some blocks are written out to * files before being written out to the output. * * @param blockSize * The block size for the source/parity files. * @param out * The destination for the reovered block. */ private void encodeFileToStream(FileSystem fs, Path sourceFile, Path parityFile, StripeReader sReader, long blockSize, OutputStream out, Progressable reporter) throws IOException { OutputStream[] tmpOuts = new OutputStream[codec.parityLength]; // One parity block can be written directly to out, rest to local files. tmpOuts[0] = out; File[] tmpFiles = new File[codec.parityLength - 1]; for (int i = 0; i < codec.parityLength - 1; i++) { tmpFiles[i] = File.createTempFile("parity", "_" + i); LOG.info("Created tmp file " + tmpFiles[i]); tmpFiles[i].deleteOnExit(); } try { // Loop over stripe int stripe = 0; while (sReader.hasNext()) { reporter.progress(); // Create input streams for blocks in the stripe. InputStream[] blocks = sReader.getNextStripeInputs(); try { // Create output streams to the temp files. for (int i = 0; i < codec.parityLength - 1; i++) { tmpOuts[i + 1] = new FileOutputStream(tmpFiles[i]); } // Call the implementation of encoding. encodeStripe(fs, sourceFile, parityFile, blocks, blockSize, tmpOuts, reporter, true, stripe); stripe++; } finally { RaidUtils.closeStreams(blocks); } // Close output streams to the temp files and write the temp files // to the output provided. for (int i = 0; i < codec.parityLength - 1; i++) { tmpOuts[i + 1].close(); tmpOuts[i + 1] = null; InputStream in = new FileInputStream(tmpFiles[i]); RaidUtils.copyBytes(in, out, writeBufs[i], blockSize); reporter.progress(); } } } finally { for (int i = 0; i < codec.parityLength - 1; i++) { if (tmpOuts[i + 1] != null) { tmpOuts[i + 1].close(); } tmpFiles[i].delete(); LOG.info("Deleted tmp file " + tmpFiles[i]); } } }
From source file:io.hops.erasure_coding.Encoder.java
License:Apache License
/** * Wraps around encodeStripeImpl in order to configure buffers. * Having buffers of the right size is extremely important. If the the * buffer size is not a divisor of the block size, we may end up reading * across block boundaries./*from www . jav a2 s .c om*/ */ void encodeStripe(FileSystem fs, Path sourceFile, Path parityFile, InputStream[] blocks, long blockSize, OutputStream[] outs, Progressable reporter, boolean computeBlockChecksum, int stripe) throws IOException { configureBuffers(blockSize); int boundedBufferCapacity = 1; ParallelStreamReader parallelReader = new ParallelStreamReader(reporter, blocks, bufSize, parallelism, boundedBufferCapacity, blockSize); parallelReader.start(); Checksum[] sourceChecksums = null; Checksum[] parityChecksums = null; if (computeBlockChecksum) { sourceChecksums = new Checksum[codec.stripeLength]; for (int i = 0; i < sourceChecksums.length; i++) { sourceChecksums[i] = new CRC32(); } parityChecksums = new Checksum[codec.parityLength]; for (int i = 0; i < parityChecksums.length; i++) { parityChecksums[i] = new CRC32(); } } try { for (long encoded = 0; encoded < blockSize; encoded += bufSize) { ParallelStreamReader.ReadResult readResult = null; try { readResult = parallelReader.getReadResult(); } catch (InterruptedException e) { throw new IOException("Interrupted while waiting for read result"); } // Cannot tolerate any IO errors. IOException readEx = readResult.getException(); if (readEx != null) { throw readEx; } if (computeBlockChecksum) { updateChecksums(sourceChecksums, readResult.readBufs); } code.encodeBulk(readResult.readBufs, writeBufs); reporter.progress(); // Now that we have some data to write, send it to the temp files. for (int i = 0; i < codec.parityLength; i++) { outs[i].write(writeBufs[i], 0, bufSize); if (computeBlockChecksum) { parityChecksums[i].update(writeBufs[i], 0, bufSize); } reporter.progress(); } } sendChecksums((DistributedFileSystem) fs, sourceFile, sourceChecksums, stripe, codec.stripeLength); sendChecksums((DistributedFileSystem) fs, parityFile, parityChecksums, stripe, codec.parityLength); } finally { parallelReader.shutdown(); } }