List of usage examples for org.apache.hadoop.io SequenceFile createWriter
@Deprecated public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, int bufferSize, short replication, long blockSize, boolean createParent, CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException
From source file:org.apache.accumulo.server.logger.LogWriter.java
License:Apache License
@Override public LogCopyInfo startCopy(TInfo info, AuthInfo credentials, final String localLog, final String fullyQualifiedFileName, final boolean sort) { log.info("Copying " + localLog + " to " + fullyQualifiedFileName); final long t1 = System.currentTimeMillis(); try {/*w w w . jav a 2 s . c om*/ Long id = file2id.get(localLog); if (id != null) close(info, id); } catch (NoSuchLogIDException e) { log.error("Unexpected error thrown", e); throw new RuntimeException(e); } File file; try { file = new File(findLocalFilename(localLog)); log.info(file.getAbsoluteFile().toString()); } catch (FileNotFoundException ex) { throw new RuntimeException(ex); } long result = file.length(); copyThreadPool.execute(new Runnable() { @Override public void run() { Thread.currentThread().setName("Copying " + localLog + " to shared file system"); for (int i = 0; i < 3; i++) { try { if (sort) { copySortLog(localLog, fullyQualifiedFileName); } else { copyLog(localLog, fullyQualifiedFileName); } return; } catch (IOException e) { log.error("error during copy", e); } UtilWaitThread.sleep(1000); } log.error("Unable to copy file to DFS, too many retries " + localLog); try { fs.create(new Path(fullyQualifiedFileName + ".failed")).close(); } catch (IOException ex) { log.error("Unable to create failure flag file", ex); } long t2 = System.currentTimeMillis(); if (metrics.isEnabled()) metrics.add(LogWriterMetrics.copy, (t2 - t1)); } private void copySortLog(String localLog, String fullyQualifiedFileName) throws IOException { final long SORT_BUFFER_SIZE = acuConf.getMemoryInBytes(Property.LOGGER_SORT_BUFFER_SIZE); FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(fs.getConf()).getRaw()); Path dest = new Path(fullyQualifiedFileName + ".recovered"); log.debug("Sorting log file to DSF " + dest); fs.mkdirs(dest); int part = 0; Reader reader = new SequenceFile.Reader(local, new Path(findLocalFilename(localLog)), fs.getConf()); try { final ArrayList<Pair<LogFileKey, LogFileValue>> kv = new ArrayList<Pair<LogFileKey, LogFileValue>>(); long memorySize = 0; while (true) { final long position = reader.getPosition(); final LogFileKey key = new LogFileKey(); final LogFileValue value = new LogFileValue(); try { if (!reader.next(key, value)) break; } catch (EOFException e) { log.warn("Unexpected end of file reading write ahead log " + localLog); break; } kv.add(new Pair<LogFileKey, LogFileValue>(key, value)); memorySize += reader.getPosition() - position; if (memorySize > SORT_BUFFER_SIZE) { writeSortedEntries(dest, part++, kv); kv.clear(); memorySize = 0; } } if (!kv.isEmpty()) writeSortedEntries(dest, part++, kv); fs.create(new Path(dest, "finished")).close(); } finally { reader.close(); } } private void writeSortedEntries(Path dest, int part, final List<Pair<LogFileKey, LogFileValue>> kv) throws IOException { String path = dest + String.format("/part-r-%05d", part); log.debug("Writing partial log file to DSF " + path); log.debug("Sorting"); Span span = Trace.start("Logger sort"); span.data("logfile", dest.getName()); Collections.sort(kv, new Comparator<Pair<LogFileKey, LogFileValue>>() { @Override public int compare(Pair<LogFileKey, LogFileValue> o1, Pair<LogFileKey, LogFileValue> o2) { return o1.getFirst().compareTo(o2.getFirst()); } }); span.stop(); span = Trace.start("Logger write"); span.data("logfile", dest.getName()); MapFile.Writer writer = new MapFile.Writer(fs.getConf(), fs, path, LogFileKey.class, LogFileValue.class); short replication = (short) acuConf.getCount(Property.LOGGER_RECOVERY_FILE_REPLICATION); fs.setReplication(new Path(path + "/" + MapFile.DATA_FILE_NAME), replication); fs.setReplication(new Path(path + "/" + MapFile.INDEX_FILE_NAME), replication); try { for (Pair<LogFileKey, LogFileValue> entry : kv) writer.append(entry.getFirst(), entry.getSecond()); } finally { writer.close(); span.stop(); } } private void copyLog(final String localLog, final String fullyQualifiedFileName) throws IOException { Path dest = new Path(fullyQualifiedFileName + ".copy"); log.debug("Copying log file to DSF " + dest); fs.delete(dest, true); LogFileKey key = new LogFileKey(); LogFileValue value = new LogFileValue(); Writer writer = null; Reader reader = null; try { short replication = (short) acuConf.getCount(Property.LOGGER_RECOVERY_FILE_REPLICATION); writer = SequenceFile.createWriter(fs, fs.getConf(), dest, LogFileKey.class, LogFileValue.class, fs.getConf().getInt("io.file.buffer.size", 4096), replication, fs.getDefaultBlockSize(), SequenceFile.CompressionType.BLOCK, new DefaultCodec(), null, new Metadata()); FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(fs.getConf()).getRaw()); reader = new SequenceFile.Reader(local, new Path(findLocalFilename(localLog)), fs.getConf()); while (reader.next(key, value)) { writer.append(key, value); } } catch (IOException ex) { log.warn("May have a partial copy of a recovery file: " + localLog, ex); } finally { if (reader != null) reader.close(); if (writer != null) writer.close(); } // Make file appear in the shared file system as the target name only after it is completely copied fs.rename(dest, new Path(fullyQualifiedFileName)); log.info("Copying " + localLog + " complete"); } }); return new LogCopyInfo(result, null); }
From source file:org.apache.avro.hadoop.io.AvroSequenceFile.java
License:Apache License
/** * Creates a writer from a set of options. * * <p>Since there are different implementations of <code>Writer</code> depending on the * compression type, this method constructs the appropriate subclass depending on the * compression type given in the <code>options</code>.</p> * * @param options The options for the writer. * @return A new writer instance./*w ww . ja va 2s .c om*/ * @throws IOException If the writer cannot be created. */ public static SequenceFile.Writer createWriter(Writer.Options options) throws IOException { return SequenceFile.createWriter(options.getFileSystem(), options.getConfigurationWithAvroSerialization(), options.getOutputPath(), options.getKeyClass(), options.getValueClass(), options.getBufferSizeBytes(), options.getReplicationFactor(), options.getBlockSizeBytes(), options.getCompressionType(), options.getCompressionCodec(), options.getProgressable(), options.getMetadataWithAvroSchemas()); }