Example usage for org.apache.hadoop.fs Path getParent

List of usage examples for org.apache.hadoop.fs Path getParent

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path computeCacheChecksumPath(Path cachePath) {
    Path parent = cachePath.getParent();
    String name = String.format("%s.acrc", cachePath.getName()); //$NON-NLS-1$
    return new Path(parent, name);
}

From source file:com.asakusafw.testdriver.file.FileExporterRetriever.java

License:Apache License

@Override
public void truncate(FileExporterDescription description, TestContext context) throws IOException {
    LOG.info("cleaning output files: {}", description);
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    FileSystem fs = path.getFileSystem(config);
    Path output = path.getParent();
    Path target;/*w  w w  . ja v  a  2s. c  om*/
    if (output == null) {
        LOG.warn(
                "?????????: {}",
                path);
        target = fs.makeQualified(path);
    } else {
        LOG.warn("??????: {}", output);
        target = fs.makeQualified(output);
    }
    LOG.debug("start removing file: {}", target);
    boolean succeed = fs.delete(target, true);
    LOG.debug("finish removing file (succeed={}): {}", succeed, target);
}

From source file:com.asakusafw.testdriver.mapreduce.io.TemporaryOutputRetriever.java

License:Apache License

@Override
public void truncate(TemporaryOutputDescription description, TestContext context) throws IOException {
    LOG.debug("Deleting output directory: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    Path output = path.getParent();
    Path target;//from w ww.j  av  a 2 s  .  com
    if (output == null) {
        LOG.warn(MessageFormat.format(Messages.getString("TemporaryOutputRetriever.warnDeleteBaseDirectory"), //$NON-NLS-1$
                path));
        target = fs.makeQualified(path);
    } else {
        LOG.debug("output directory will be deleted: {}", output); //$NON-NLS-1$
        target = fs.makeQualified(output);
    }
    TemporaryInputPreparator.delete(fs, target);
}

From source file:com.asakusafw.testdriver.temporary.TemporaryOutputRetriever.java

License:Apache License

@Override
public void truncate(TemporaryOutputDescription description, TestContext context) throws IOException {
    LOG.debug("Deleting output directory: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    Path output = path.getParent();
    Path target;// w  ww. j  av a 2s  .  c  om
    if (output == null) {
        LOG.warn("Skipped deleting output directory because it is a base directory: {}", path);
        target = fs.makeQualified(path);
    } else {
        LOG.debug("Output directory will be deleted: {}", output); //$NON-NLS-1$
        target = fs.makeQualified(output);
    }
    TemporaryInputPreparator.delete(fs, target);
}

From source file:com.baifendian.swordfish.common.hadoop.HdfsClient.java

License:Apache License

/**
 * ???/*from   ww  w . jav a  2s .co m*/
 */
public void setPermission(Path path, FsPermission perm) throws IOException {
    fileSystem.setPermission(path, perm);
    if (path.getParent() != null) {
        setPermission(path.getParent(), perm);
    }
}

From source file:com.blackberry.logdriver.LockedFs.java

License:Apache License

@SuppressWarnings("deprecation")
public void move(Configuration conf, String[] from, String to) throws IOException {
    FileSystem fs = FileSystem.get(conf);

    List<FileStatus> fromList = new ArrayList<FileStatus>();
    for (String s : from) {
        FileStatus[] statuses = fs.globStatus(new Path(s));
        if (statuses == null) {
            continue;
        }/*from ww  w . j  a  va 2  s  .  c o m*/
        for (FileStatus status : statuses) {
            fromList.add(status);
        }
    }

    Path toPath = new Path(to);
    Boolean toExists = fs.exists(toPath);
    FileStatus toFileStatus = null;
    if (toExists) {
        toFileStatus = fs.getFileStatus(toPath);
    }

    // If there is no from, that's a problem.
    if (fromList.isEmpty()) {
        throw new IOException("No input files found");
    }

    // If the to exists, and is a file, that's a problem too.
    if (toExists && !toFileStatus.isDir()) {
        throw new IOException("Destination file exists:" + to);
    }

    // If the destination exists, and is a directory, then ensure that none of
    // the from list names will clash with existing contents of the directory.
    if (toExists && toFileStatus.isDir()) {
        for (FileStatus fromStatus : fromList) {
            String name = fromStatus.getPath().getName();
            if (fs.exists(new Path(toPath, name))) {
                throw new IOException("Destination file exists:" + to + "/" + name);
            }
        }
    }

    // If the destination doesn't exist, but it ends with a slash, then create
    // it as a directory.
    if (!toExists && to.endsWith("/")) {
        fs.mkdirs(toPath);
        toFileStatus = fs.getFileStatus(toPath);
        toExists = true;
    }

    // If the destination doesn't exist, and there is more than one 'from', then
    // create a directory.
    if (!toExists && fromList.size() > 1) {
        fs.mkdirs(toPath);
        toFileStatus = fs.getFileStatus(toPath);
    }

    // If there was only one from, then just rename it to to
    if (fromList.size() == 1) {
        fs.mkdirs(toPath.getParent());
        fs.rename(fromList.get(0).getPath(), toPath);
    }

    // If there was more than one from, then for each file in the from list,
    // move it to the to directory.
    if (fromList.size() > 1) {
        for (FileStatus fromStatus : fromList) {
            String name = fromStatus.getPath().getName();
            fs.rename(fromStatus.getPath(), new Path(toPath, name));
        }
    }
}

From source file:com.blm.orc.OrcInputFormat.java

License:Apache License

@Override
public RowReader<OrcStruct> getReader(InputSplit inputSplit, Options options) throws IOException {
    final OrcSplit split = (OrcSplit) inputSplit;
    final Path path = split.getPath();
    Path root;//  w  ww  .  ja  v  a2s  .c  o  m
    if (split.hasBase()) {
        if (split.isOriginal()) {
            root = path.getParent();
        } else {
            root = path.getParent().getParent();
        }
    } else {
        root = path;
    }
    final Path[] deltas = AcidUtils.deserializeDeltas(root, split.getDeltas());
    final Configuration conf = options.getConfiguration();
    final Reader reader;
    final int bucket;
    Reader.Options readOptions = new Reader.Options();
    readOptions.range(split.getStart(), split.getLength());
    if (split.hasBase()) {
        bucket = AcidUtils.parseBaseBucketFilename(split.getPath(), conf).getBucket();
        reader = OrcFile.createReader(path, OrcFile.readerOptions(conf));
        final List<OrcProto.Type> types = reader.getTypes();
        setIncludedColumns(readOptions, types, conf, split.isOriginal());
        setSearchArgument(readOptions, types, conf, split.isOriginal());
    } else {
        bucket = (int) split.getStart();
        reader = null;
    }
    String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY, Long.MAX_VALUE + ":");
    ValidTxnList validTxnList = new ValidTxnListImpl(txnString);
    final OrcRawRecordMerger records = new OrcRawRecordMerger(conf, true, reader, split.isOriginal(), bucket,
            validTxnList, readOptions, deltas);
    return new RowReader<OrcStruct>() {
        OrcStruct innerRecord = records.createValue();

        @Override
        public ObjectInspector getObjectInspector() {
            return ((StructObjectInspector) records.getObjectInspector()).getAllStructFieldRefs()
                    .get(OrcRecordUpdater.ROW).getFieldObjectInspector();
        }

        @Override
        public boolean next(RecordIdentifier recordIdentifier, OrcStruct orcStruct) throws IOException {
            boolean result;
            // filter out the deleted records
            do {
                result = records.next(recordIdentifier, innerRecord);
            } while (result && OrcRecordUpdater.getOperation(innerRecord) == OrcRecordUpdater.DELETE_OPERATION);
            if (result) {
                // swap the fields with the passed in orcStruct
                orcStruct.linkFields(OrcRecordUpdater.getRow(innerRecord));
            }
            return result;
        }

        @Override
        public RecordIdentifier createKey() {
            return records.createKey();
        }

        @Override
        public OrcStruct createValue() {
            return new OrcStruct(records.getColumns());
        }

        @Override
        public long getPos() throws IOException {
            return records.getPos();
        }

        @Override
        public void close() throws IOException {
            records.close();
        }

        @Override
        public float getProgress() throws IOException {
            return records.getProgress();
        }
    };
}

From source file:com.bonc.mr_roamRecognition_hjpt.comm.PathCombineTextInputFormat.java

License:Apache License

public synchronized static List<PathFilter> getPoll() {

    List<PathFilter> pools = new ArrayList<PathFilter>();
    Map<String, String> map = ProvUtil.getCode();

    for (Map.Entry<String, String> entry : map.entrySet()) {
        final String prov_id = entry.getValue();
        pools.add(new PathFilter() {
            String provId = prov_id;

            @Override/*  w  ww .  j a  v a2  s.co  m*/
            public boolean accept(Path path) {
                String fileName = path.getParent().toString();
                boolean need = fileName.endsWith(prov_id);
                return need;
            }
        });
    }

    return pools;
}

From source file:com.ceph.rados.fs.hdfs.RadosFileSystem.java

License:Apache License

/**
 * @param permission Currently ignored.//from w  w w . ja va 2s .  c  om
 */
@Override
public boolean mkdirs(Path path, FsPermission permission) throws IOException {
    Path absolutePath = makeAbsolute(path);
    List<Path> paths = new ArrayList<Path>();
    do {
        paths.add(0, absolutePath);
        absolutePath = absolutePath.getParent();
    } while (absolutePath != null);

    boolean result = true;
    for (Path p : paths) {
        result &= mkdir(p);
    }
    return result;
}

From source file:com.ceph.rados.fs.hdfs.RadosFileSystem.java

License:Apache License

/**
 * @param permission Currently ignored.//from  www  .ja  v  a 2s .c o  m
 */
@Override
public FSDataOutputStream create(Path file, FsPermission permission, boolean overwrite, int bufferSize,
        short replication, long blockSize, Progressable progress) throws IOException {

    INode inode = store.retrieveINode(makeAbsolute(file));
    if (inode != null) {
        if (overwrite) {
            delete(file, true);
        } else {
            throw new FileAlreadyExistsException("File already exists: " + file);
        }
    } else {
        Path parent = file.getParent();
        if (parent != null) {
            if (!mkdirs(parent)) {
                throw new IOException("Mkdirs failed to create " + parent.toString());
            }
        }
    }
    return new FSDataOutputStream(new RadosHDFSOutputStream(store, makeAbsolute(file).toString()));
}