Example usage for org.apache.hadoop.fs Path SEPARATOR

List of usage examples for org.apache.hadoop.fs Path SEPARATOR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<Boolean> delete(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        HDFSResource resource) {/*  ww w  . j a va  2s  .  c o m*/
    final String filePath = resource.dirPath + Path.SEPARATOR + resource.fileName;
    LOG.info("{}deleting file:{}", new Object[] { logPrefix, filePath });
    try {
        Result<Boolean> result = ugi.doAs(new PrivilegedExceptionAction<Result<Boolean>>() {
            @Override
            public Result<Boolean> run() {
                try (FileSystem fs = FileSystem.get(hdfsEndpoint.hdfsConfig)) {
                    fs.delete(new Path(filePath), false);
                    return Result.success(true);
                } catch (IOException ex) {
                    LOG.warn("{}could not delete file:{}", logPrefix, ex.getMessage());
                    return Result.externalUnsafeFailure(new HDFSException("hdfs file delete", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalUnsafeFailure(new HDFSException("hdfs file delete", ex));
    }
}

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<Boolean> simpleCreate(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        final HDFSResource hdfsResource) {
    final String filePath = hdfsResource.dirPath + Path.SEPARATOR + hdfsResource.fileName;
    LOG.info("{}creating file:{}", new Object[] { logPrefix, filePath });
    try {/*ww  w  .  j a v  a 2  s . co m*/
        Result<Boolean> result = ugi.doAs(new PrivilegedExceptionAction<Result<Boolean>>() {
            @Override
            public Result<Boolean> run() {
                try (FileSystem fs = FileSystem.get(hdfsEndpoint.hdfsConfig)) {
                    if (!fs.isDirectory(new Path(hdfsResource.dirPath))) {
                        fs.mkdirs(new Path(hdfsResource.dirPath));
                    }
                    if (fs.isFile(new Path(filePath))) {
                        return Result.success(false);
                    }
                    try (FSDataOutputStream out = fs.create(new Path(filePath), (short) 1)) {
                        return Result.success(true);
                    }
                } catch (IOException ex) {
                    LOG.warn("{}could not write file:{}", logPrefix, ex.getMessage());
                    return Result.externalUnsafeFailure(new HDFSException("hdfs file simpleCreate", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalUnsafeFailure(new HDFSException("hdfs file simpleCreate", ex));
    }
}

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<Boolean> createWithLength(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        final HDFSResource hdfsResource, final long fileSize) {
    final String filePath = hdfsResource.dirPath + Path.SEPARATOR + hdfsResource.fileName;
    LOG.debug("{}creating file:{}", new Object[] { logPrefix, filePath });
    try {//from   w w  w. ja  va2s  . c  o m
        Result<Boolean> result = ugi.doAs(new PrivilegedExceptionAction<Result<Boolean>>() {
            @Override
            public Result<Boolean> run() {
                try (FileSystem fs = FileSystem.get(hdfsEndpoint.hdfsConfig)) {
                    if (!fs.isDirectory(new Path(hdfsResource.dirPath))) {
                        fs.mkdirs(new Path(hdfsResource.dirPath));
                    }
                    if (fs.isFile(new Path(filePath))) {
                        return Result.success(false);
                    }
                    Random rand = new Random(1234);
                    try (FSDataOutputStream out = fs.create(new Path(filePath))) {
                        for (int i = 0; i < fileSize / 1024; i++) {
                            byte[] data = new byte[1024];
                            rand.nextBytes(data);
                            out.write(data);
                            out.flush();
                        }
                        if (fileSize % 1024 != 0) {
                            byte[] data = new byte[(int) (fileSize % 1024)];
                            rand.nextBytes(data);
                            out.write(data);
                            out.flush();
                        }
                        return Result.success(true);
                    }
                } catch (IOException ex) {
                    LOG.warn("{}could not create file:{}", logPrefix, ex.getMessage());
                    return Result.externalUnsafeFailure(new HDFSException("hdfs file createWithLength", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalUnsafeFailure(new HDFSException("hdfs file createWithLength", ex));
    }
}

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<byte[]> read(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        HDFSResource resource, final KRange readRange) {
    final String filePath = resource.dirPath + Path.SEPARATOR + resource.fileName;
    LOG.debug("{}reading from file:{}", new Object[] { logPrefix, filePath });
    try {//w w w.j  a  v  a  2 s .co m
        Result<byte[]> result = ugi.doAs(new PrivilegedExceptionAction<Result<byte[]>>() {
            @Override
            public Result<byte[]> run() {
                try (DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(hdfsEndpoint.hdfsConfig);
                        FSDataInputStream in = fs.open(new Path(filePath))) {
                    int readLength = (int) (readRange.upperAbsEndpoint() - readRange.lowerAbsEndpoint() + 1);
                    byte[] byte_read = new byte[readLength];
                    in.readFully(readRange.lowerAbsEndpoint(), byte_read);
                    return Result.success(byte_read);
                } catch (IOException ex) {
                    LOG.warn("{}could not read file:{} ex:{}",
                            new Object[] { logPrefix, filePath, ex.getMessage() });
                    return Result.externalSafeFailure(new HDFSException("hdfs file read", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalSafeFailure(new HDFSException("hdfs file read", ex));
    }
}

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<Boolean> append(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        HDFSResource resource, final byte[] data) {
    final String filePath = resource.dirPath + Path.SEPARATOR + resource.fileName;
    LOG.debug("{}appending to file:{}", new Object[] { logPrefix, filePath });
    try {/*w  ww . ja v a  2s.  c o m*/
        Result<Boolean> result = ugi.doAs(new PrivilegedExceptionAction<Result<Boolean>>() {
            @Override
            public Result<Boolean> run() {
                try (DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(hdfsEndpoint.hdfsConfig);
                        FSDataOutputStream out = fs.append(new Path(filePath))) {
                    out.write(data);
                    out.flush();
                    return Result.success(true);
                } catch (IOException ex) {
                    LOG.warn("{}could not append to file:{} ex:{}",
                            new Object[] { logPrefix, filePath, ex.getMessage() });
                    return Result.externalUnsafeFailure(new HDFSException("hdfs file append", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalUnsafeFailure(new HDFSException("hdfs file append", ex));
    }
}

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<ManifestJSON> readManifest(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        HDFSResource hdfsResource) {/*from ww w .  j  a  v a 2s.c om*/
    final String filePath = hdfsResource.dirPath + Path.SEPARATOR + hdfsResource.fileName;
    LOG.debug("{}reading manifest:{}", new Object[] { logPrefix, filePath });
    try {
        Result<ManifestJSON> result = ugi.doAs(new PrivilegedExceptionAction<Result<ManifestJSON>>() {
            @Override
            public Result<ManifestJSON> run() {
                try (DistributedFileSystem fs = (DistributedFileSystem) FileSystem
                        .get(hdfsEndpoint.hdfsConfig)) {
                    if (!fs.isFile(new Path(filePath))) {
                        LOG.warn("{}file does not exist", new Object[] { logPrefix, filePath });
                        return Result.externalSafeFailure(new HDFSException("hdfs file read"));
                    }
                    try (FSDataInputStream in = fs.open(new Path(filePath))) {
                        long manifestLength = fs.getLength(new Path(filePath));
                        byte[] manifestByte = new byte[(int) manifestLength];
                        in.readFully(manifestByte);
                        ManifestJSON manifest = ManifestHelper.getManifestJSON(manifestByte);
                        return Result.success(manifest);
                    }
                } catch (IOException ex) {
                    LOG.warn("{}could not read file:{} ex:{}",
                            new Object[] { logPrefix, filePath, ex.getMessage() });
                    return Result.externalSafeFailure(new HDFSException("hdfs file read", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalSafeFailure(new HDFSException("hdfs file read", ex));
    }
}

From source file:se.sics.nstream.hops.hdfs.HDFSHelper.java

License:Open Source License

public static Result<Boolean> writeManifest(UserGroupInformation ugi, final HDFSEndpoint hdfsEndpoint,
        final HDFSResource hdfsResource, final ManifestJSON manifest) {
    final String filePath = hdfsResource.dirPath + Path.SEPARATOR + hdfsResource.fileName;
    LOG.debug("{}writing manifest:{}", new Object[] { logPrefix, filePath });
    try {/*from  w w w.  j  a va  2 s .c  o m*/
        Result<Boolean> result = ugi.doAs(new PrivilegedExceptionAction<Result<Boolean>>() {
            @Override
            public Result<Boolean> run() {
                try (FileSystem fs = FileSystem.get(hdfsEndpoint.hdfsConfig)) {
                    if (!fs.isDirectory(new Path(hdfsResource.dirPath))) {
                        fs.mkdirs(new Path(hdfsResource.dirPath));
                    }
                    if (fs.isFile(new Path(filePath))) {
                        return Result.success(false);
                    }
                    try (FSDataOutputStream out = fs.create(new Path(filePath))) {
                        byte[] manifestByte = ManifestHelper.getManifestByte(manifest);
                        out.write(manifestByte);
                        out.flush();
                        return Result.success(true);
                    }
                } catch (IOException ex) {
                    LOG.warn("{}could not create file:{}", logPrefix, ex.getMessage());
                    return Result.externalUnsafeFailure(new HDFSException("hdfs file createWithLength", ex));
                }
            }
        });
        LOG.trace("{}op completed", new Object[] { logPrefix });
        return result;
    } catch (IOException | InterruptedException ex) {
        LOG.error("{}unexpected exception:{}", logPrefix, ex);
        return Result.externalUnsafeFailure(new HDFSException("hdfs file createWithLength", ex));
    }
}

From source file:skewtune.mapreduce.STJobTracker.java

License:Apache License

public static String getUserDir(String user) {
    return SUBDIR + Path.SEPARATOR + user;
}

From source file:skewtune.mapreduce.STJobTracker.java

License:Apache License

public static String getLocalJobDir(String user, String jobid) {
    return getUserDir(user) + Path.SEPARATOR + jobid;
}

From source file:skewtune.mapreduce.STJobTracker.java

License:Apache License

public static String getLocalJobTokenFile(String user, String jobid) {
    return getLocalJobDir(user, jobid) + Path.SEPARATOR + JOB_TOKEN_FILE;
}