Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.ibm.jaql.io.hadoop.DirectFileOutputCommiter.java

License:Apache License

@Override
public void setupJob(JobContext context) throws IOException {
    // Create the path to the file, if needed.
    JobConf conf = context.getJobConf();
    Path outputPath = FileOutputFormat.getOutputPath(conf);
    if (outputPath != null) {
        Path tmpDir = outputPath.getParent();
        FileSystem fileSys = outputPath.getFileSystem(conf);
        if (!fileSys.mkdirs(outputPath.getParent())) {
            throw new IOException("Mkdirs failed to create " + tmpDir.toString());
        }/*from w w  w  . ja  v a  2 s .  com*/
    }
}

From source file:com.ibm.jaql.io.hadoop.FileOutputConfigurator.java

License:Apache License

public void setSequential(JobConf conf) throws Exception {
    registerSerializers(conf);/*from  w ww.j  a  v a2  s . c o  m*/

    // For an expression, the location is the final file name
    Path outPath = new Path(location);
    FileSystem fs = outPath.getFileSystem(conf);
    outPath = outPath.makeQualified(fs);
    if (fs.exists(outPath)) {
        // TODO: Jaql currently has overwrite semantics; add flag to control this
        if (fs.isFile(outPath)) {
            fs.delete(outPath, false);
        } else {
            // Look for a map-reduce output directory
            FileStatus[] nonMR = fs.listStatus(outPath, new PathFilter() {
                boolean onlyOne = true;

                public boolean accept(Path path) {
                    String name = path.getName();
                    if (name.matches("([.][.]?)|([.]part-[0-9]+.crc)|(part-[0-9]+)")) {
                        return false;
                    }
                    if (onlyOne) {
                        onlyOne = false;
                        return true;
                    }
                    return false;
                }
            });
            if (nonMR.length > 0) {
                throw new IOException(
                        "directory exists and is not a map-reduce output directory: " + nonMR[0].getPath());
            }
            fs.delete(outPath, true);
        }
    }

    // In sequential mode, we will write directly to the output file
    // and bypass the _temporary directory and rename of the standard 
    // FileOutputCommitter by using our own DirectFileOutputCommitter.
    FileOutputFormat.setOutputPath(conf, outPath.getParent());
    conf.setClass("mapred.output.committer.class", DirectFileOutputCommiter.class, OutputCommitter.class);
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

/**
 * Merge between two paths/*from  w  w w  . ja  va 2 s .  c  om*/
 *
 * @param hostName
 * @param p path
 * @param objectKey
 * @return merged path
 */
private String getMergedPath(String hostName, Path p, String objectKey) {
    if ((p.getParent() != null) && (p.getName() != null) && (p.getParent().toString().equals(hostName))) {
        if (objectKey.equals(p.getName())) {
            return p.toString();
        }
        return hostName + objectKey;
    }
    return hostName + objectKey;
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

/**
 * Extracts from the object key an unified object name or name without task ID
 *
 * @param objectKey/*from w  w w .  j  a  v a  2 s  . c o m*/
 * @param isUnifiedObjectKey
 * @return
 */
private String extractFromObjectKeyWithTaskID(String objectKey, boolean isUnifiedObjectKey) {
    Path p = new Path(objectKey);
    int index = objectKey.indexOf("-" + HADOOP_ATTEMPT);
    if (index > 0) {
        String attempt = objectKey.substring(objectKey.lastIndexOf("-") + 1);
        try {
            TaskAttemptID.forName(attempt);
            if (isUnifiedObjectKey) {
                return p.getParent().toString();
            } else {
                return objectKey.substring(0, index);
            }
        } catch (IllegalArgumentException e) {
            return objectKey;
        }
    } else if (isUnifiedObjectKey && objectKey.indexOf(HADOOP_SUCCESS) > 0) {
        return p.getParent().toString();
    }
    return objectKey;
}

From source file:com.ibm.stocator.fs.cos.COSLocalDirAllocator.java

License:Apache License

public File createTmpFileForWrite(String pathStr, long size, Configuration conf) throws IOException {
    String sha256hex = DigestUtils.sha256Hex(pathStr);
    Path path = getLocalPathForWrite(sha256hex, size, conf, true);
    File tmpDir = new File(path.getParent().toUri().getPath());
    File tmpFile = new File(tmpDir, sha256hex + UUID.randomUUID().toString());
    return tmpFile;
}

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

@Override
public boolean delete(Path f, boolean recursive) throws IOException {
    LOG.debug("delete method: {}. recursive {}", f.toString(), recursive);
    String objNameModified = getObjectNameRoot(f, HADOOP_TEMPORARY, true);
    LOG.debug("Modified object name {}", objNameModified);
    if (objNameModified.contains(HADOOP_TEMPORARY)) {
        return true;
    }/*from   www  . ja  v a2  s.c om*/
    Path pathToObj = new Path(objNameModified);
    if (f.getName().startsWith(HADOOP_ATTEMPT)) {
        FileStatus[] fsList = storageClient.list(hostNameScheme, pathToObj.getParent(), true);
        if (fsList.length > 0) {
            for (FileStatus fs : fsList) {
                if (fs.getPath().getName().endsWith(f.getName())) {
                    storageClient.delete(hostNameScheme, fs.getPath(), recursive);
                }
            }
        }
    } else {
        FileStatus[] fsList = storageClient.list(hostNameScheme, pathToObj, true);
        if (fsList.length > 0) {
            for (FileStatus fs : fsList) {
                storageClient.delete(hostNameScheme, fs.getPath(), recursive);
            }
        }
    }
    return true;
}

From source file:com.ibm.stocator.fs.ObjectStoreFileSystem.java

License:Open Source License

/**
 * {@inheritDoc}//from  www.  ja  va 2  s . c o m
 *
 * When path is of the form schema://dataroot.provider/objectname/_temporary/0
 * it is assumed that new job started to write it's data.
 * In this case we create an empty object schema://dataroot.provider/objectname
 * that will later be used to identify objects that were created by Spark.
 * This is needed for fault tolerance coverage to identify data that was created
 * by failed jobs or tasks.
 * dataroot/object created as a 0 size object with type application/directory
 *
 * @param f path to create
 * @return boolean on success or failure
 * @throws IOException
 */
@Override
public boolean mkdirs(Path f) throws IOException {
    LOG.debug("mkdirs: {}", f.toString());
    if (f.getParent().toString().endsWith(HADOOP_TEMPORARY)) {
        String objNameModified = getObjectNameRoot(f, HADOOP_TEMPORARY, true);
        Path pathToObj = new Path(objNameModified);
        String plainObjName = pathToObj.getParent().toString();
        LOG.debug("Going to create identifier {}", plainObjName);
        Map<String, String> metadata = new HashMap<String, String>();
        metadata.put("Data-Origin", "stocator");
        FSDataOutputStream outStream = storageClient.createObject(plainObjName, "application/directory",
                metadata, statistics);
        outStream.close();
    }
    return true;
}

From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java

License:Open Source License

/**
 * Merge between two paths//from   w  w  w  .  ja va 2  s  .c om
 *
 * @param hostName
 * @param p path
 * @param objectName
 * @return merged path
 */
private String getMergedPath(String hostName, Path p, String objectName) {
    if ((p.getParent() != null) && (p.getName() != null) && (p.getParent().toString().equals(hostName))) {
        if (objectName.equals(p.getName())) {
            return p.toString();
        }
        if (objectName.startsWith(p.getName())) {
            return p.getParent() + objectName;
        }
        return p.toString();
    }
    return hostName + objectName;
}

From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java

License:Open Source License

/**
 * Accepts any object name./*from   w w w .  j ava2  s .c  o m*/
 * If object name of the form
 * a/b/c/gil.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c
 *    .csv-attempt_20160317132a_wrong_0000_m_000000_1
 * Then a/b/c/gil.data is returned.
 * Code testing that attempt_20160317132a_wrong_0000_m_000000_1 is valid
 * task id identifier
 *
 * @param objectName
 * @return unified object name
 */
private String extractUnifiedObjectName(String objectName) {
    Path p = new Path(objectName);
    if (objectName.indexOf("-" + HADOOP_ATTEMPT) > 0) {
        String attempt = objectName.substring(objectName.lastIndexOf("-") + 1);
        try {
            TaskAttemptID.forName(attempt);
            return p.getParent().toString();
        } catch (IllegalArgumentException e) {
            return objectName;
        }
    } else if (objectName.indexOf(HADOOP_SUCCESS) > 0) {
        return p.getParent().toString();
    }
    return objectName;
}

From source file:com.ibm.stocator.fs.swift2d.systemtests.SwiftTestUtils.java

License:Open Source License

/**
 * Assert that a path exists -but make no assertions as to the
 * type of that entry//from  w  w w  . j a  va 2  s.co m
 *
 * @param fileSystem filesystem to examine
 * @param message message to include in the assertion failure message
 * @param path path in the filesystem
 * @throws IOException IO problems
 */
public static void assertPathExists(FileSystem fileSystem, String message, Path path) throws IOException {
    if (!fileSystem.exists(path)) {
        //failure, report it
        fail(message + ": not found " + path + " in " + path.getParent());
    }
}