List of usage examples for org.apache.hadoop.io MapFile INDEX_FILE_NAME
String INDEX_FILE_NAME
To view the source code for org.apache.hadoop.io MapFile INDEX_FILE_NAME.
Click Source Link
From source file:com.foursquare.twofishes.io.MapFileConcurrentReader.java
License:Apache License
protected synchronized void open(Path dir, WritableComparator comparator, final Configuration conf, final SequenceFile.Reader.Option... options) throws IOException { final Path dataFile = new Path(dir, MapFile.DATA_FILE_NAME); final Path indexFile = new Path(dir, MapFile.INDEX_FILE_NAME); // open the data this.data = new ThreadLocal<SequenceFile.Reader>() { protected SequenceFile.Reader initialValue() { try { SequenceFile.Reader r = createDataFileReader(dataFile, conf, options); LOG.info("opened new SequenceFile.Reader for " + dataFile); synchronized (this) { allDataFiles.add(r); }/*from w ww. j a v a 2 s .co m*/ return r; } catch (IOException ioe) { throw new RuntimeException(ioe); } } }; this.firstPosition = data.get().getPosition(); this.comparator = WritableComparator.get(data.get().getKeyClass().asSubclass(WritableComparable.class)); // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
From source file:com.google.mr4c.sources.MapFileSource.java
License:Open Source License
public MapFileSource(FileSystem fs, Path dir) throws IOException { m_fs = fs;// www .j a va2 s . co m m_config = m_fs.getConf(); Path root = new Path(fs.getUri()); m_dir = new Path(root, dir); m_dirStr = m_dir.toUri().getPath(); m_dataPath = new Path(m_dir, MapFile.DATA_FILE_NAME); m_indexPath = new Path(m_dir, MapFile.INDEX_FILE_NAME); m_metaPath = new Path(m_dir, "metadata"); }
From source file:kogiri.common.hadoop.io.reader.map.IndexCloseableMapFileReader.java
License:Apache License
protected synchronized void open(FileSystem fs, String dirName, WritableComparator comparator, Configuration conf) throws IOException { Path dir = new Path(dirName); Path dataFile = new Path(dir, MapFile.DATA_FILE_NAME); Path indexFile = new Path(dir, MapFile.INDEX_FILE_NAME); // open the data this.data = createDataFileReader(fs, dataFile, conf); this.firstPosition = data.getPosition(); if (comparator == null) { this.comparator = WritableComparator.get(data.getKeyClass().asSubclass(WritableComparable.class)); } else {// w w w. j a va 2 s. co m this.comparator = comparator; } // open the index this.index = new SequenceFile.Reader(fs, indexFile, conf); }
From source file:org.apache.accumulo.core.file.map.MapFileUtil.java
License:Apache License
public static SequenceFile.Reader openIndex(Configuration conf, FileSystem fs, Path mapFile) throws IOException { Path indexPath = new Path(mapFile, MapFile.INDEX_FILE_NAME); SequenceFile.Reader index = null; try {/* w ww. j a v a 2 s. co m*/ index = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(indexPath))); return index; } catch (IOException e) { throw e; } }
From source file:org.apache.accumulo.server.logger.LogWriter.java
License:Apache License
@Override public LogCopyInfo startCopy(TInfo info, AuthInfo credentials, final String localLog, final String fullyQualifiedFileName, final boolean sort) { log.info("Copying " + localLog + " to " + fullyQualifiedFileName); final long t1 = System.currentTimeMillis(); try {/*from ww w . j a v a 2s .c om*/ Long id = file2id.get(localLog); if (id != null) close(info, id); } catch (NoSuchLogIDException e) { log.error("Unexpected error thrown", e); throw new RuntimeException(e); } File file; try { file = new File(findLocalFilename(localLog)); log.info(file.getAbsoluteFile().toString()); } catch (FileNotFoundException ex) { throw new RuntimeException(ex); } long result = file.length(); copyThreadPool.execute(new Runnable() { @Override public void run() { Thread.currentThread().setName("Copying " + localLog + " to shared file system"); for (int i = 0; i < 3; i++) { try { if (sort) { copySortLog(localLog, fullyQualifiedFileName); } else { copyLog(localLog, fullyQualifiedFileName); } return; } catch (IOException e) { log.error("error during copy", e); } UtilWaitThread.sleep(1000); } log.error("Unable to copy file to DFS, too many retries " + localLog); try { fs.create(new Path(fullyQualifiedFileName + ".failed")).close(); } catch (IOException ex) { log.error("Unable to create failure flag file", ex); } long t2 = System.currentTimeMillis(); if (metrics.isEnabled()) metrics.add(LogWriterMetrics.copy, (t2 - t1)); } private void copySortLog(String localLog, String fullyQualifiedFileName) throws IOException { final long SORT_BUFFER_SIZE = acuConf.getMemoryInBytes(Property.LOGGER_SORT_BUFFER_SIZE); FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(fs.getConf()).getRaw()); Path dest = new Path(fullyQualifiedFileName + ".recovered"); log.debug("Sorting log file to DSF " + dest); fs.mkdirs(dest); int part = 0; Reader reader = new SequenceFile.Reader(local, new Path(findLocalFilename(localLog)), fs.getConf()); try { final ArrayList<Pair<LogFileKey, LogFileValue>> kv = new ArrayList<Pair<LogFileKey, LogFileValue>>(); long memorySize = 0; while (true) { final long position = reader.getPosition(); final LogFileKey key = new LogFileKey(); final LogFileValue value = new LogFileValue(); try { if (!reader.next(key, value)) break; } catch (EOFException e) { log.warn("Unexpected end of file reading write ahead log " + localLog); break; } kv.add(new Pair<LogFileKey, LogFileValue>(key, value)); memorySize += reader.getPosition() - position; if (memorySize > SORT_BUFFER_SIZE) { writeSortedEntries(dest, part++, kv); kv.clear(); memorySize = 0; } } if (!kv.isEmpty()) writeSortedEntries(dest, part++, kv); fs.create(new Path(dest, "finished")).close(); } finally { reader.close(); } } private void writeSortedEntries(Path dest, int part, final List<Pair<LogFileKey, LogFileValue>> kv) throws IOException { String path = dest + String.format("/part-r-%05d", part); log.debug("Writing partial log file to DSF " + path); log.debug("Sorting"); Span span = Trace.start("Logger sort"); span.data("logfile", dest.getName()); Collections.sort(kv, new Comparator<Pair<LogFileKey, LogFileValue>>() { @Override public int compare(Pair<LogFileKey, LogFileValue> o1, Pair<LogFileKey, LogFileValue> o2) { return o1.getFirst().compareTo(o2.getFirst()); } }); span.stop(); span = Trace.start("Logger write"); span.data("logfile", dest.getName()); MapFile.Writer writer = new MapFile.Writer(fs.getConf(), fs, path, LogFileKey.class, LogFileValue.class); short replication = (short) acuConf.getCount(Property.LOGGER_RECOVERY_FILE_REPLICATION); fs.setReplication(new Path(path + "/" + MapFile.DATA_FILE_NAME), replication); fs.setReplication(new Path(path + "/" + MapFile.INDEX_FILE_NAME), replication); try { for (Pair<LogFileKey, LogFileValue> entry : kv) writer.append(entry.getFirst(), entry.getSecond()); } finally { writer.close(); span.stop(); } } private void copyLog(final String localLog, final String fullyQualifiedFileName) throws IOException { Path dest = new Path(fullyQualifiedFileName + ".copy"); log.debug("Copying log file to DSF " + dest); fs.delete(dest, true); LogFileKey key = new LogFileKey(); LogFileValue value = new LogFileValue(); Writer writer = null; Reader reader = null; try { short replication = (short) acuConf.getCount(Property.LOGGER_RECOVERY_FILE_REPLICATION); writer = SequenceFile.createWriter(fs, fs.getConf(), dest, LogFileKey.class, LogFileValue.class, fs.getConf().getInt("io.file.buffer.size", 4096), replication, fs.getDefaultBlockSize(), SequenceFile.CompressionType.BLOCK, new DefaultCodec(), null, new Metadata()); FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(fs.getConf()).getRaw()); reader = new SequenceFile.Reader(local, new Path(findLocalFilename(localLog)), fs.getConf()); while (reader.next(key, value)) { writer.append(key, value); } } catch (IOException ex) { log.warn("May have a partial copy of a recovery file: " + localLog, ex); } finally { if (reader != null) reader.close(); if (writer != null) writer.close(); } // Make file appear in the shared file system as the target name only after it is completely copied fs.rename(dest, new Path(fullyQualifiedFileName)); log.info("Copying " + localLog + " complete"); } }); return new LogCopyInfo(result, null); }