List of usage examples for org.apache.hadoop.fs FileSystem getDefaultReplication
@Deprecated public short getDefaultReplication()
From source file:com.bigdata.mapreduce.seqtotext.beta.ZipFileRecordReader.java
License:Apache License
/** * Initialise and open the ZIP file from the FileSystem *//*from w w w.ja v a 2 s.c o m*/ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { // Configuration conf = new Configuration(); // conf.set("fs.defaultFS", "hdfs://training.musigma.com:8020/user/musigma/"); FileSplit split = (FileSplit) inputSplit; System.out.println("the task attempt instance is : " + taskAttemptContext.getJobName()); System.out.println("the task attempt instance is : " + taskAttemptContext.getWorkingDirectory().toString()); Configuration conf = taskAttemptContext.getConfiguration(); Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); System.out.println("file system replication : " + fs.getDefaultReplication()); // Open the stream fsin = fs.open(path); // zip = new ZipInputStream(fsin); tar = new TarInputStream(fsin); System.out.println("tar input stream is : " + tar.toString()); }
From source file:com.bigdata.mapreduce.seqtotext.beta1.ZipFileRecordReader.java
License:Apache License
public ZipFileRecordReader(Configuration conf, org.apache.hadoop.mapred.FileSplit split) throws IOException, InterruptedException { Path path = split.getPath();/*from w ww .jav a2 s. c om*/ FileSystem fs = path.getFileSystem(conf); System.out.println("file system replication : " + fs.getDefaultReplication()); // Open the stream fsin = fs.open(path); // zip = new ZipInputStream(fsin); tar = new TarInputStream(fsin); System.out.println("tar input stream is : " + tar.toString()); }
From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java
License:Apache License
private static short getReplicationFactor(EnumSet<FileAttribute> fileAttributes, FileStatus sourceFile, FileSystem targetFS) { return fileAttributes.contains(FileAttribute.REPLICATION) ? sourceFile.getReplication() : targetFS.getDefaultReplication(); }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
private static void touchFile(String path) throws Exception { FileSystem fs; DataOutputStream outputStream = null; GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfiguration()); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = null; try {/*from www . j a v a2 s .c o m*/ fs = cluster.getFileSystem(); final Path qualifiedPath = new Path(path).makeQualified(fs); final long blockSize = fs.getDefaultBlockSize() * 2; outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize); compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor); Message msg = new Message("generating test data".getBytes()); AuditUtil.attachHeaders(msg, currentTimestamp); byte[] encodeMsg = Base64.encodeBase64(msg.getData().array()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); // Genearate a msg with different timestamp. Default window period is 60sec AuditUtil.attachHeaders(msg, nextMinuteTimeStamp); encodeMsg = Base64.encodeBase64(msg.getData().array()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); compressedOut.flush(); compressedOut.close(); pathList.add(qualifiedPath); ++nFiles; FileStatus fileStatus = fs.getFileStatus(qualifiedPath); System.out.println(fileStatus.getBlockSize()); System.out.println(fileStatus.getReplication()); } finally { compressedOut.close(); IOUtils.cleanup(null, outputStream); CodecPool.returnCompressor(gzipCompressor); } }
From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java
License:Apache License
private static void touchFile(String path) throws Exception { FileSystem fs; DataOutputStream outputStream = null; GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfigurationForCluster()); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = null; try {// w w w . j ava2s .c o m fs = cluster.getFileSystem(); final Path qualifiedPath = new Path(path).makeQualified(fs); final long blockSize = fs.getDefaultBlockSize() * 2; outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize); compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor); compressedOut.write(new byte[FILE_SIZE]); compressedOut.write("\n".getBytes()); compressedOut.flush(); //outputStream.write(new byte[FILE_SIZE]); pathList.add(qualifiedPath); } finally { compressedOut.close(); IOUtils.cleanup(null, outputStream); CodecPool.returnCompressor(gzipCompressor); } }
From source file:dima.kmeansseq.SequenceFile.java
License:Apache License
/** * Construct the preferred type of SequenceFile Writer. * //w ww. j a v a2s . c o m * @param fs * The configured filesystem. * @param conf * The configuration. * @param name * The name of the file. * @param keyClass * The 'key' type. * @param valClass * The 'value' type. * @param compressionType * The compression type. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException */ public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType) throws IOException { return createWriter(fs, conf, name, keyClass, valClass, fs.getConf().getInt("io.file.buffer.size", 4096), fs.getDefaultReplication(), fs.getDefaultBlockSize(), compressionType, new DefaultCodec(), null, new Metadata()); }
From source file:dima.kmeansseq.SequenceFile.java
License:Apache License
/** * Construct the preferred type of SequenceFile Writer. * /*from www . ja va2 s. c o m*/ * @param fs * The configured filesystem. * @param conf * The configuration. * @param name * The name of the file. * @param keyClass * The 'key' type. * @param valClass * The 'value' type. * @param compressionType * The compression type. * @param progress * The Progressable object to track progress. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException */ public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType, Progressable progress) throws IOException { return createWriter(fs, conf, name, keyClass, valClass, fs.getConf().getInt("io.file.buffer.size", 4096), fs.getDefaultReplication(), fs.getDefaultBlockSize(), compressionType, new DefaultCodec(), progress, new Metadata()); }
From source file:dima.kmeansseq.SequenceFile.java
License:Apache License
/** * Construct the preferred type of SequenceFile Writer. * /*from w w w .j a v a 2 s . c om*/ * @param fs * The configured filesystem. * @param conf * The configuration. * @param name * The name of the file. * @param keyClass * The 'key' type. * @param valClass * The 'value' type. * @param compressionType * The compression type. * @param codec * The compression codec. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException */ public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec) throws IOException { return createWriter(fs, conf, name, keyClass, valClass, fs.getConf().getInt("io.file.buffer.size", 4096), fs.getDefaultReplication(), fs.getDefaultBlockSize(), compressionType, codec, null, new Metadata()); }
From source file:dima.kmeansseq.SequenceFile.java
License:Apache License
/** * Construct the preferred type of SequenceFile Writer. * // w w w . j av a 2s . c om * @param fs * The configured filesystem. * @param conf * The configuration. * @param name * The name of the file. * @param keyClass * The 'key' type. * @param valClass * The 'value' type. * @param compressionType * The compression type. * @param codec * The compression codec. * @param progress * The Progressable object to track progress. * @param metadata * The metadata of the file. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException */ public static Writer createWriter(FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Progressable progress, Metadata metadata) throws IOException { return createWriter(fs, conf, name, keyClass, valClass, fs.getConf().getInt("io.file.buffer.size", 4096), fs.getDefaultReplication(), fs.getDefaultBlockSize(), compressionType, codec, progress, metadata); }
From source file:edu.bigdata.training.fileformats.compress.SequenceFileWriter.java
public static void main(String[] args) throws IOException { String uri = "output"; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path path = new Path(uri); IntWritable key = new IntWritable(); Text value = new Text(); File infile = new File("src/main/resources/input.txt"); SequenceFile.Writer writer = null; try {/* w ww.j a v a 2s. co m*/ writer = SequenceFile.createWriter(conf, Writer.file(path), Writer.keyClass(key.getClass()), Writer.valueClass(value.getClass()), Writer.bufferSize(fs.getConf().getInt("io.file.buffer.size", 4096)), Writer.replication(fs.getDefaultReplication()), Writer.blockSize(1073741824), Writer.compression(SequenceFile.CompressionType.BLOCK, new DefaultCodec()), Writer.progressable(null), Writer.metadata(new Metadata())); int ctr = 100; List<String> lines = FileUtils.readLines(infile); for (String line : lines) { key.set(ctr++); value.set(line); if (ctr < 150) { System.out.printf("[%s]\t%s\t%s\n", writer.getLength(), key, value); } writer.append(key, value); } } finally { IOUtils.closeStream(writer); } }