Example usage for org.apache.hadoop.fs FileSystem create

List of usage examples for org.apache.hadoop.fs FileSystem create

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. create

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem create.

Prototype

public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException

Source Link

Document

Create an FSDataOutputStream at the indicated Path.

Usage

From source file:alluxio.underfs.hdfs.HdfsUnderFileSystem.java

License:Apache License

@Override
public FSDataOutputStream create(String path, CreateOptions options) throws IOException {
    IOException te = null;/*  www . j  ava 2s  .co  m*/
    RetryPolicy retryPolicy = new CountingRetry(MAX_TRY);
    Permission perm = options.getPermission();
    while (retryPolicy.attemptRetry()) {
        try {
            LOG.debug("Creating HDFS file at {} with perm {}", path, perm.toString());
            // TODO(chaomin): support creating HDFS files with specified block size and replication.
            return FileSystem.create(mFileSystem, new Path(path), new FsPermission(perm.getMode().toShort()));
        } catch (IOException e) {
            LOG.error("Retry count {} : {} ", retryPolicy.getRetryCount(), e.getMessage(), e);
            te = e;
        }
    }
    throw te;
}

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopPut.java

License:Apache License

private OutputStream getOutput(FileSystem fs, Path path) throws IOException {
    if (RuntimeContext.get().isSimulation()) {
        return new VoidOutputStream();
    } else {/*from  w  w w .  ja  v  a2  s .co m*/
        return fs.create(path, true, BUFFER_SIZE);
    }
}

From source file:com.bigjob.Client.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, int appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    LOG.debug("HDFS Destination for Script: " + dst.toString());
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {//from  w  w  w .  ja  v a 2  s.co  m
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:com.btoddb.chronicle.apps.AvroTools.java

License:Open Source License

private void testFileAndFix(Path inFile) throws IOException {
    FileContext context = FileContext.getFileContext(hdfsConfig);
    AvroFSInput input = new AvroFSInput(context, inFile);

    ReflectDatumReader<Object> reader = new ReflectDatumReader<>();
    FileReader<Object> fileReader = DataFileReader.openReader(input, reader);

    Path outFile = inFile.suffix(".fixing");
    FSDataOutputStream output = FileSystem.create(outFile.getFileSystem(hdfsConfig), outFile,
            FsPermission.getDefault());/*from   w w  w.  j  a  va2 s  . co  m*/
    DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>());
    writer.setCodec(CodecFactory.snappyCodec());

    boolean corrupted = false;
    long count = 0;

    try {
        Schema schema = fileReader.getSchema();
        writer.create(schema, output);

        for (;;) {
            try {
                if (fileReader.hasNext()) {
                    Object obj = fileReader.next();
                    count++;
                    writer.append(obj);
                } else {
                    break;
                }
            } catch (AvroRuntimeException e) {
                corrupted = true;
                System.out.println("  - file pointer = " + input.tell());
                if (e.getCause() instanceof EOFException) {
                    System.out.println("  - EOF occurred so we're done : " + e.getMessage());
                    break;
                } else if (e.getCause() instanceof IOException) {
                    System.out.println("  - will try to 'next' past the error : " + e.getMessage());
                    try {
                        fileReader.next();
                        System.out.println("  - 'next' worked - didn't really expect it to, but great!");
                    } catch (Exception e2) {
                        System.out.println("  - 'next' did not work - will continue on and see what happens : "
                                + e2.getMessage());
                    }
                    continue;
                }
                break;
            } catch (Exception e) {
                corrupted = true;
                System.out.println("  - file pointer = " + input.tell());
                e.printStackTrace();
                break;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        System.out.println(("  - processed " + count + " records"));
        if (null != fileReader) {
            try {
                fileReader.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        if (null != writer) {
            try {
                writer.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    if (!corrupted) {
        outFile.getFileSystem(hdfsConfig).delete(outFile, false);
    } else {
        outFile.getFileSystem(hdfsConfig).rename(outFile, inFile.suffix(".fixed"));
    }
}

From source file:com.datatorrent.stram.util.FSJsonLineFile.java

License:Apache License

public FSJsonLineFile(Path path, FsPermission permission) throws IOException {
    fs = FileSystem.newInstance(path.toUri(), new Configuration());
    FSDataOutputStream myos;//from w w  w  .j  a  v  a 2s  .c  o  m
    if (fs.exists(path)) {
        try {
            // happens if not the first application attempt
            myos = fs.append(path);
        } catch (IOException ex) {
            LOG.warn("Caught exception (OK during unit test): {}", ex.getMessage());
            myos = FileSystem.create(fs, path, permission);
        }
    } else {
        myos = FileSystem.create(fs, path, permission);
    }
    os = myos;
    this.objectMapper = (new JSONSerializationProvider()).getContext(null);
}

From source file:com.epam.hadoop.nv.yarn.Client.java

License:Apache License

private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {/*from w  w  w . j  a v a 2 s. c  o m*/
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:com.github.hdl.tensorflow.yarn.app.TFContainer.java

License:Apache License

public void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {

    execCmd("pwd");
    execCmd("ls -l");
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    LOG.info("copy: " + fileSrcPath + " ===> " + dst.toString());
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {/*from w  w w.ja v a  2 s  .  c om*/
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }

    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(URL.fromURI(dst.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:com.inmobi.conduit.distcp.tools.mapred.lib.TestDynamicInputFormat.java

License:Apache License

private static void createFile(String path) throws Exception {
    FileSystem fileSystem = null;
    DataOutputStream outputStream = null;
    try {/*  w w  w.java2 s . c  o  m*/
        fileSystem = cluster.getFileSystem();
        outputStream = fileSystem.create(new Path(path), true, 0);
        expectedFilePaths.add(fileSystem.listStatus(new Path(path))[0].getPath().toString());
    } finally {
        IOUtils.cleanup(null, fileSystem, outputStream);
    }
}

From source file:com.inmobi.conduit.distcp.tools.mapred.TestUniformSizeInputFormat.java

License:Apache License

private static int createFile(String path, int fileSize) throws Exception {
    FileSystem fileSystem = null;
    DataOutputStream outputStream = null;
    try {/*from w ww  .j a  va  2s.c  o m*/
        fileSystem = cluster.getFileSystem();
        outputStream = fileSystem.create(new Path(path), true, 0);
        int size = (int) Math.ceil(fileSize + (1 - random.nextFloat()) * fileSize);
        outputStream.write(new byte[size]);
        return size;
    } finally {
        IOUtils.cleanup(null, fileSystem, outputStream);
    }
}

From source file:com.inmobi.conduit.distcp.tools.TestGlobbedCopyListing.java

License:Apache License

private static void touchFile(String path) throws Exception {
    FileSystem fileSystem = null;
    DataOutputStream outputStream = null;
    try {//w w  w  .j a  v  a2s  . c o  m
        fileSystem = cluster.getFileSystem();
        outputStream = fileSystem.create(new Path(path), true, 0);
        recordInExpectedValues(path);
    } finally {
        IOUtils.cleanup(null, fileSystem, outputStream);
    }
}