List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f, short replication) throws IOException
From source file:cn.lhfei.hadoop.ch03.FileCopyWithProgress.java
License:Apache License
public static void main(String[] args) { String localSrc = args[0];//from w w w. j a va2 s . c o m String dst = args[1]; FileSystem fs = null; InputStream in = null; OutputStream out = null; try { Configuration conf = new Configuration(); fs = FileSystem.get(URI.create(localSrc), conf); in = new BufferedInputStream(new FileInputStream(localSrc)); out = fs.create(new Path(dst), new Progressable() { @Override public void progress() { log.info("... ..."); } }); IOUtils.copyBytes(in, out, 4096, true); } catch (FileNotFoundException e) { // e.printStackTrace(); log.error(e.getMessage(), e); } catch (IOException e) { // e.printStackTrace(); log.error(e.getMessage(), e); } finally { IOUtils.closeStream(in); IOUtils.closeStream(out); } }
From source file:co.nubetech.hiho.common.HihoTestCase.java
License:Apache License
public void createTextFileInHDFS(String inputData, String filePath, String nameOfFile) throws IOException { FileSystem fs = getFileSystem(); FSDataOutputStream out = null;//from www . j a v a 2s .c o m Path inputFile = new Path(filePath + "/" + nameOfFile); try { out = fs.create(inputFile, false); out.write(inputData.getBytes(), 0, inputData.getBytes().length); out.close(); out = null; // Cheking input file exists or not. Path inputPath = new Path(fs.getHomeDirectory(), filePath + "/" + nameOfFile); assertTrue(fs.exists(inputPath)); } finally { if (out != null) { out.close(); } } }
From source file:co.nubetech.hiho.mapreduce.lib.output.NoKeyOnlyValueOutputFormat.java
License:Apache License
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException { boolean isCompressed = getCompressOutput(context); Configuration conf = context.getConfiguration(); String ext = ""; CompressionCodec codec = null;/*www .j a v a 2 s. c o m*/ if (isCompressed) { // create the named codec Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(context, GzipCodec.class); codec = ReflectionUtils.newInstance(codecClass, conf); ext = codec.getDefaultExtension(); } Path file = getDefaultWorkFile(context, ext); FileSystem fs = file.getFileSystem(conf); FSDataOutputStream fileOut = fs.create(file, false); DataOutputStream ostream = fileOut; if (isCompressed) { ostream = new DataOutputStream(codec.createOutputStream(fileOut)); } return new NoKeyRecordWriter<K, V>(ostream); }
From source file:com.alexholmes.hadooputils.sort.DelimitedTextOutputFormat.java
License:Apache License
public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { SortConfig sortConf = new SortConfig(job); boolean isCompressed = getCompressOutput(job); String lineSeparator = sortConf.getRowSeparator("\n"); byte[] hexcode = SortConfig.getHexDelimiter(lineSeparator); lineSeparator = (hexcode != null) ? new String(hexcode, "UTF-8") : lineSeparator; if (!isCompressed) { Path file = FileOutputFormat.getTaskOutputPath(job, name); FileSystem fs = file.getFileSystem(job); FSDataOutputStream fileOut = fs.create(file, progress); return new DelimitedLineRecordWriter<K, V>(fileOut, lineSeparator); } else {//from w w w .j a va2s . c o m Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class); CompressionCodec codec = ReflectionUtils.newInstance(codecClass, job); Path file = FileOutputFormat.getTaskOutputPath(job, name + codec.getDefaultExtension()); FileSystem fs = file.getFileSystem(job); FSDataOutputStream fileOut = fs.create(file, progress); return new DelimitedLineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)), lineSeparator); } }
From source file:com.alibaba.jstorm.hdfs.common.HdfsUtils.java
License:Apache License
/** * Returns null if file already exists. throws if there was unexpected problem *//*from ww w . j a va 2s . c om*/ public static FSDataOutputStream tryCreateFile(FileSystem fs, Path file) throws IOException { try { FSDataOutputStream os = fs.create(file, false); return os; } catch (FileAlreadyExistsException e) { return null; } catch (RemoteException e) { if (e.unwrapRemoteException() instanceof AlreadyBeingCreatedException) { return null; } else { // unexpected error throw e; } } }
From source file:com.asakusafw.compiler.directio.DirectFileIoProcessorRunTest.java
License:Apache License
private void put(String target, String... contents) throws IOException { FileSystem fs = FileSystem.get(tester.configuration()); try (OutputStream output = fs.create(getPath(target), true); PrintWriter w = new PrintWriter(new OutputStreamWriter(output, "UTF-8"))) { for (String line : contents) { w.println(line);/*from w w w .java 2s . co m*/ } } }
From source file:com.asakusafw.dag.runtime.directio.TransactionManager.java
License:Apache License
private void setTransactionInfo(boolean value) throws IOException { Path transactionInfo = getTransactionInfoPath(); FileSystem fs = transactionInfo.getFileSystem(configuration); if (value) {/*from w w w . jav a 2 s .c o m*/ try (OutputStream output = new SafeOutputStream(fs.create(transactionInfo, false)); PrintWriter writer = new PrintWriter( new OutputStreamWriter(output, HadoopDataSourceUtil.COMMENT_CHARSET))) { for (Map.Entry<String, String> entry : transactionProperties.entrySet()) { if (entry.getValue() != null) { writer.printf("%s: %s%n", //$NON-NLS-1$ entry.getKey(), entry.getValue()); } } } } else { fs.delete(transactionInfo, false); } }
From source file:com.asakusafw.dag.runtime.directio.TransactionManager.java
License:Apache License
private void setCommitted(boolean value) throws IOException { Path commitMark = getCommitMarkPath(); FileSystem fs = commitMark.getFileSystem(configuration); if (value) {//from w ww. j av a 2s . c o m fs.create(commitMark, false).close(); } else { fs.delete(commitMark, false); } }
From source file:com.asakusafw.lang.compiler.mapreduce.testing.mock.WritableOutputFormat.java
License:Apache License
@Override public RecordWriter<NullWritable, T> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { Path path = getDefaultWorkFile(context, null); FileSystem fs = path.getFileSystem(context.getConfiguration()); return new Writer<>(new WritableModelOutput<T>(fs.create(path, true))); }
From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java
License:Apache License
/** * Opens a temporary {@link ModelOutput} for the specified path. * @param <V> data type/*www .jav a2 s. c om*/ * @param conf configuration * @param dataType data type * @param path target path * @return the opened {@link ModelOutput} * @throws IOException if failed to open output * @throws IllegalArgumentException if some parameters were {@code null} */ @SuppressWarnings("unchecked") public static <V> ModelOutput<V> openOutput(Configuration conf, Class<V> dataType, Path path) throws IOException { if (conf == null) { throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$ } if (dataType == null) { throw new IllegalArgumentException("dataType must not be null"); //$NON-NLS-1$ } if (path == null) { throw new IllegalArgumentException("path must not be null"); //$NON-NLS-1$ } FileSystem fs = path.getFileSystem(conf); if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Opening temporary output: {0} (fs={1})", //$NON-NLS-1$ path, fs.getUri())); } if (Writable.class.isAssignableFrom(dataType)) { return (ModelOutput<V>) new TemporaryFileOutput<>(fs.create(path, true), dataType.getName(), OUTPUT_INIT_BUFFER_SIZE, OUTPUT_PAGE_SIZE); } SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, path, NullWritable.class, dataType); return new SequenceFileModelOutput<>(out); }