List of usage examples for org.apache.hadoop.fs FileSystem setVerifyChecksum
public void setVerifyChecksum(boolean verifyChecksum)
From source file:com.cloudera.HioBench.java
License:Apache License
public static void main(String[] args) throws Exception { options = new Options(); final Configuration conf = new Configuration(); if (options.dumpConf) { Configuration.dumpConfiguration(conf, new PrintWriter(System.out)); }/*from w w w. jav a 2 s. c o m*/ final FileSystem fs = FileSystem.get(new URI(options.hdfsUri), conf); fs.setVerifyChecksum(!options.skipChecksum); if (!fs.exists(options.filePath)) { System.out.println("no file at " + options.filePath + "; writing " + "new file now with length " + options.nGigsInFile + " gigs..."); writeFile(fs); System.out.println("done."); } else if (fs.getLength(options.filePath) != options.nBytesInFile) { System.out.println("existing file " + options.filename + " has length " + fs.getLength(options.filePath) + ", but we wanted length " + options.nBytesInFile + ". Re-creating."); writeFile(fs); System.out.println("done."); } else { System.out.println( "using existing file at " + options.filePath + " of length " + options.nGigsInFile + " gigs."); } long nanoStart = System.nanoTime(); WorkerThread threads[] = new WorkerThread[options.nThreads]; for (int i = 0; i < options.nThreads; i++) { threads[i] = new WorkerThread(i == 0, fs, WorkerThread.createBenchReader(options, i)); } for (int i = 0; i < options.nThreads; i++) { threads[i].start(); } for (int i = 0; i < options.nThreads; i++) { threads[i].join(); } for (int i = 0; i < options.nThreads; i++) { Throwable t = threads[i].getException(); if (t != null) { System.err.println("there were exceptions. Aborting."); System.exit(1); } } long nanoEnd = System.nanoTime(); fs.close(); long totalIo = options.nThreads; totalIo *= options.nBytesToRead; float nanoDiff = nanoEnd - nanoStart; float seconds = nanoDiff / 1000000000; System.out.println(String.format("Using %d threads, read %s in %f seconds", options.nThreads, prettyPrintByteSize(totalIo), seconds)); float rate = totalIo / seconds; System.out.println("Average rate was " + prettyPrintByteSize(rate) + "/s"); }
From source file:com.gruter.hadoop.customShell.CustomShell.java
License:Apache License
/** * Return the {@link FileSystem} specified by src and the conf. * It the {@link FileSystem} supports checksum, set verifyChecksum. */// w ww . ja v a2 s. c o m private FileSystem getSrcFileSystem(Path src, boolean verifyChecksum) throws IOException { FileSystem srcFs = src.getFileSystem(getConf()); srcFs.setVerifyChecksum(verifyChecksum); return srcFs; }
From source file:edu.purdue.cybercenter.dm.storage.HdfsStorageFileManager.java
private void initiateTrasfer(String sourceType, String source, String destType, Integer storageDestination) throws Exception { FileSystem sourceFS = getFileSystemType(sourceType); FileSystem destFS = getFileSystemType(destType); Storage destStorage = Storage.findStorage(storageDestination); // this // method/* ww w .j av a 2 s . c om*/ // is // given // the id // of the storage you get the // storage object of it destFS.setVerifyChecksum(false); // visitAllFiles2(sourceFS, source, destFS, destStorage); visitAllFiles(sourceFS, new File(source), destFS, destStorage); System.out.println("files transfered"); }
From source file:edu.purdue.cybercenter.dm.storage.HdfsStorageFileManager.java
private StorageFile initiateTransferOneFile(String sourceType, String source, String destType, Integer storageDestination) throws IOException { FileSystem sourceFS = getFileSystemType(sourceType); FileSystem destFS = getFileSystemType(destType); Storage destStorage = Storage.findStorage(storageDestination); destFS.setVerifyChecksum(false); FileId fileId = new FileId(); String dest = getStorageDesitiantion(sourceFS, source, AccessMethodType.FILE, destFS, destStorage, fileId); transferFile(sourceFS, source, destFS, dest, fileId, true); return StorageFile.findStorageFile(fileId.getFileId()); }
From source file:io.prestosql.plugin.hive.HdfsEnvironment.java
License:Apache License
public FileSystem getFileSystem(String user, Path path, Configuration configuration) throws IOException { return hdfsAuthentication.doAs(user, () -> { FileSystem fileSystem = path.getFileSystem(configuration); fileSystem.setVerifyChecksum(verifyChecksum); return fileSystem; });//from w w w .j a v a 2 s. c om }
From source file:org.apache.hawq.pxf.plugins.hdfs.ChunkRecordReader.java
License:Apache License
/** * Constructs a ChunkRecordReader instance. * * @param job the job configuration/*from w w w. j a va 2 s . c om*/ * @param split contains the file name, begin byte of the split and the * bytes length * @throws IOException if an I/O error occurs when accessing the file or * creating input stream to read from it */ public ChunkRecordReader(Configuration job, FileSplit split) throws IOException { maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE); validateLength(maxLineLength); start = split.getStart(); end = start + split.getLength(); final Path file = split.getPath(); compressionCodecs = new CompressionCodecFactory(job); codec = compressionCodecs.getCodec(file); // open the file and seek to the start of the split job.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, true); final FileSystem fs = file.getFileSystem(job); fs.setVerifyChecksum(false); fileIn = fs.open(file, ChunkReader.DEFAULT_BUFFER_SIZE); fileLength = getInputStream().getFileLength(); if (isCompressedInput()) { decompressor = CodecPool.getDecompressor(codec); if (codec instanceof SplittableCompressionCodec) { final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream( fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK); in = new ChunkReader(cIn); start = cIn.getAdjustedStart(); end = cIn.getAdjustedEnd(); filePosition = cIn; // take pos from compressed stream } else { in = new ChunkReader(codec.createInputStream(fileIn, decompressor)); filePosition = fileIn; } } else { fileIn.seek(start); in = new ChunkReader(fileIn); filePosition = fileIn; } /* * If this is not the first split, we always throw away first record * because we always (except the last split) read one extra line in * next() method. */ if (start != 0) { start += in.readLine(new ChunkWritable(), maxBytesToConsume(start)); } this.pos = start; }
From source file:org.springframework.data.hadoop.fs.FsShell.java
License:Apache License
public void copyToLocal(boolean ignorecrc, boolean crc, String src, String localdst) { File dst = new File(localdst); Path srcpath = new Path(src); try {/*from w w w.j ava 2s .c o m*/ FileSystem srcFs = getFS(srcpath); srcFs.setVerifyChecksum(ignorecrc); if (crc && !(srcFs instanceof ChecksumFileSystem)) { crc = false; } FileStatus[] srcs = srcFs.globStatus(srcpath); boolean dstIsDir = dst.isDirectory(); if (srcs.length > 1 && !dstIsDir) { throw new IllegalArgumentException( "When copying multiple files, " + "destination should be a directory."); } for (FileStatus status : srcs) { Path p = status.getPath(); File f = dstIsDir ? new File(dst, p.getName()) : dst; copyToLocal(srcFs, p, f, crc); } } catch (IOException ex) { throw new HadoopException("Cannot copy resources " + ex.getMessage(), ex); } }