Example usage for org.apache.hadoop.fs Path getParent

List of usage examples for org.apache.hadoop.fs Path getParent

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.github.sakserv.sequencefile.SequenceFileWriter.java

License:Apache License

public static void main(String[] args) {

    String outputFile = args[0];//from   w  ww  .j a  v  a 2 s.c o m

    Configuration conf = new Configuration();
    try {
        FileSystem fs = FileSystem.get(conf);

        Path seqFilePath = new Path(outputFile);
        fs.mkdirs(seqFilePath.getParent());

        SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(seqFilePath),
                SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(IntWritable.class));

        writer.append(new Text("key1"), new IntWritable(1));
        writer.append(new Text("key2"), new IntWritable(2));

        writer.close();

        LOG.info("SUCCESS: Successfully wrote " + seqFilePath + " to HDFS.");
    } catch (IOException e) {
        LOG.error("ERROR: Could not load hadoop configuration");
        e.printStackTrace();
    }

}

From source file:com.github.seqware.queryengine.tutorial.Poster.java

License:Open Source License

private void recordSpace(String key) throws IOException {
    try {/*from w  ww  .j  ava 2s .  c om*/
        Configuration conf = new Configuration();
        HBaseStorage.configureHBaseConfig(conf);
        HBaseConfiguration.addHbaseResources(conf);
        FileSystem fs = FileSystem.get(conf);
        Path homeDirectory = fs.getHomeDirectory();
        Path root = homeDirectory.getParent().getParent();
        Path hbase = new Path(root, "hbase");
        ContentSummary contentSummary = fs.getContentSummary(hbase);
        long spaceConsumedinGB = convertToGB(contentSummary);
        keyValues.put(key + "-total-space-in-GB", Long.toString(spaceConsumedinGB));

        /**
         *
         * if (spaceConsumedinGB > CUT_OFF){ return; }
         *
         */
        Path featureTable = new Path(hbase,
                Constants.Term.NAMESPACE.getTermValue(String.class) + ".hbaseTestTable_v2.Feature." + HG_19);
        contentSummary = fs.getContentSummary(featureTable);
        spaceConsumedinGB = convertToGB(contentSummary);
        keyValues.put(key + "-feature-space-in-GB", Long.toString(spaceConsumedinGB));
    } catch (FileNotFoundException e) {
        /**
         * throw away, this is ok the first time *
         */
    }

}

From source file:com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemTestBase.java

License:Open Source License

/**
 * We override certain methods in FileSystem simply to provide debug tracing. (Search for
 * "Overridden functions for debug tracing" in GoogleHadoopFileSystemBase.java).
 * We do not add or update any functionality for such methods. The following
 * tests simply exercise that path to ensure coverage. Consequently, they do not
 * really test any functionality./*from w ww.j  a v  a2 s .co  m*/
 *
 * Having coverage for these methods lets us easily determine the amount of
 * coverage that is missing in the rest of the code.
 */
@Test
public void provideCoverageForUnmodifiedMethods() throws IOException {
    // -------------------------------------------------------
    // Create test data.

    // Temporary file in GHFS.
    URI tempFileUri = GoogleCloudStorageFileSystemIntegrationTest.getTempFilePath();
    Path tempFilePath = ghfsHelper.castAsHadoopPath(tempFileUri);
    Path tempDirPath = tempFilePath.getParent();
    String text = "Hello World!";
    ghfsHelper.writeFile(tempFilePath, text, 1, false);

    // Another temporary file in GHFS.
    URI tempFileUri2 = GoogleCloudStorageFileSystemIntegrationTest.getTempFilePath();
    Path tempFilePath2 = ghfsHelper.castAsHadoopPath(tempFileUri2);

    // Temporary file in local FS.
    File localTempFile = File.createTempFile("ghfs-test-", null);
    Path localTempFilePath = new Path(localTempFile.getPath());
    Path localTempDirPath = localTempFilePath.getParent();

    // -------------------------------------------------------
    // Call methods to provide coverage for. Note that we do not attempt to
    // test their functionality as we are not testing Hadoop engine here.
    try {
        ghfs.deleteOnExit(tempFilePath);
        ghfs.getContentSummary(tempFilePath);
        ghfs.getDelegationToken("foo");
        ghfs.copyFromLocalFile(false, true, localTempFilePath, tempDirPath);
        ghfs.copyFromLocalFile(false, true, new Path[] { localTempFilePath }, tempDirPath);
        localTempFile.delete();
        ghfs.copyToLocalFile(true, tempFilePath, localTempDirPath);
        File localCopiedFile = new File(localTempDirPath.toString(), tempFilePath.getName());
        localCopiedFile.delete();
        Path localOutputPath = ghfs.startLocalOutput(tempFilePath2, localTempFilePath);
        FileWriter writer = new FileWriter(localOutputPath.toString());
        writer.write(text);
        writer.close();
        ghfs.completeLocalOutput(tempFilePath2, localOutputPath);
        ghfs.getUsed();
        ghfs.setVerifyChecksum(false);
        ghfs.getFileChecksum(tempFilePath2);
        ghfs.setPermission(tempFilePath2, FsPermission.getDefault());
        try {
            ghfs.setOwner(tempFilePath2, "foo-user", "foo-group");
        } catch (IOException ioe) {
            // Some filesystems (like the LocalFileSystem) are strict about existence of owners.
            // TODO(user): Abstract out the behaviors around owners/permissions and properly test
            // the different behaviors between different filesystems.
        }
        ghfs.setTimes(tempFilePath2, 0, 0);
    } finally {
        // We do not need to separately delete the temp files created in GHFS because
        // we delete all test buckets recursively at the end of the tests.
        if (localTempFile.exists()) {
            localTempFile.delete();
        }
    }
}

From source file:com.google.cloud.hadoop.fs.gcs.GoogleHadoopSyncableOutputStream.java

License:Open Source License

/**
 * Returns URI to be used for the next "tail" file in the series.
 *//* w  ww .  j  a  v  a 2s . c  o  m*/
private URI getNextTemporaryPath() {
    Path basePath = ghfs.getHadoopPath(finalGcsPath);
    Path baseDir = basePath.getParent();
    Path tempPath = new Path(baseDir, String.format("%s%s.%d.%s", TEMPFILE_PREFIX, basePath.getName(),
            curComponentIndex, UUID.randomUUID().toString()));
    return ghfs.getGcsPath(tempPath);
}

From source file:com.hdfs.concat.crush.Crush.java

License:Apache License

/**
 * Moves all crush input files to {@link #dest} and then moves the crush output file to {@link #srcDir}.
 *///from  www.  ja  v  a2s  . c om
private void swap(List<Path> crushInput, String crushFileName) throws IOException {

    if (crushInput.isEmpty()) {
        return;
    }

    print(Verbosity.INFO, format("\n\nSwapping %s", crushFileName));

    List<Path> movedSrc = new ArrayList<Path>(crushInput.size());
    List<Path> movedDest = new ArrayList<Path>(crushInput.size());

    Path crushedDir = crushInput.get(0).getParent();

    boolean crushFileNotInstalled = true;

    try {
        /*
         * Move each source file into the clone directory, replacing the root with the path of the clone dir.
         */
        for (Iterator<Path> iter = crushInput.iterator(); iter.hasNext();) {
            Path source = iter.next();

            /*
             * Remove the leading slash from the input file to create a path relative to the clone dir.
             */
            Path destPath = new Path(dest, source.toString().substring(1));

            rename(source, destPath.getParent(), null);

            iter.remove();

            movedSrc.add(source);
            movedDest.add(destPath);
        }

        /*
         * Install the crush output file now that all the source files have been moved to the clone dir. Sometimes the compression
         * codec messes with the names so watch out.
         */
        Path crushFile = new Path(crushFileName);

        rename(crushFile, crushedDir, null);

        crushFileNotInstalled = false;

    } finally {
        if (!crushInput.isEmpty()) {
            /*
             * We failed while moving the source files to the clone directory.
             */
            LOG.error(format(
                    "Failed while moving files into the clone directory and before installing the crush output file (%d moved and %d remaining)",
                    movedSrc.size(), crushInput.size()));

            StringBuilder sb = new StringBuilder("hadoop fs -mv ");

            for (int i = 0; i < movedSrc.size(); i++) {
                sb.append(" ");
                sb.append(movedDest.get(i));
            }

            sb.append(" ");
            sb.append(crushedDir);

            LOG.error("Execute the following to restore the file system to a good state: " + sb.toString());
        } else if (crushFileNotInstalled) {
            /*
             * We failed moving the crush output file to the source directory.
             */
            LOG.error(format("Failed while moving crush output file (%s) to the source directory (%s)",
                    crushFileName, crushedDir));
        }
    }
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static ArrayList<Byte> getInputMatrixIndexesInMapper(JobConf job) throws IOException {
    String[] matrices = job.getStrings(INPUT_MATRICIES_DIRS_CONFIG);
    String str = job.get(MAPFUNC_INPUT_MATRICIES_INDEXES_CONFIG);
    byte[] indexes;
    if (str == null || str.isEmpty()) {
        indexes = new byte[matrices.length];
        for (int i = 0; i < indexes.length; i++)
            indexes[i] = (byte) i;
    } else {/*from   w w w  .j  a  va  2s.  com*/
        String[] strs = str.split(Instruction.INSTRUCTION_DELIM);
        indexes = new byte[strs.length];
        for (int i = 0; i < strs.length; i++)
            indexes[i] = Byte.parseByte(strs[i]);
    }

    int numMatrices = matrices.length;
    if (numMatrices > Byte.MAX_VALUE)
        throw new RuntimeException("number of matrices is too large > " + Byte.MAX_VALUE);
    for (int i = 0; i < matrices.length; i++)
        matrices[i] = new Path(matrices[i]).toString();

    FileSystem fs = FileSystem.get(job);
    Path thisFile = new Path(job.get("map.input.file")).makeQualified(fs);

    //Path p=new Path(thisFileName);

    Path thisDir = thisFile.getParent().makeQualified(fs);
    ArrayList<Byte> representativeMatrixes = new ArrayList<Byte>();
    for (int i = 0; i < matrices.length; i++) {
        Path p = new Path(matrices[i]).makeQualified(fs);
        if (thisFile.toUri().compareTo(p.toUri()) == 0 || thisDir.toUri().compareTo(p.toUri()) == 0)
            representativeMatrixes.add(indexes[i]);
    }
    return representativeMatrixes;
}

From source file:com.ibm.crail.hdfs.CrailHadoopFileSystem.java

License:Apache License

@Override
public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize,
        short replication, long blockSize, Progressable progress) throws IOException {
    CrailFile fileInfo = null;/*  w ww . j av a  2s.  c o  m*/
    try {
        fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT,
                CrailLocationClass.PARENT).get().asFile();
    } catch (Exception e) {
        if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
            fileInfo = null;
        } else {
            throw new IOException(e);
        }
    }

    if (fileInfo == null) {
        Path parent = path.getParent();
        this.mkdirs(parent, FsPermission.getDirDefault());
        try {
            fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT,
                    CrailLocationClass.PARENT).get().asFile();
        } catch (Exception e) {
            throw new IOException(e);
        }
    }

    CrailBufferedOutputStream outputStream = null;
    if (fileInfo != null) {
        try {
            fileInfo.syncDir();
            outputStream = fileInfo.getBufferedOutputStream(Integer.MAX_VALUE);
        } catch (Exception e) {
            throw new IOException(e);
        }
    }

    if (outputStream != null) {
        return new CrailHDFSOutputStream(outputStream, statistics);
    } else {
        throw new IOException("Failed to create file, path " + path.toString());
    }
}

From source file:com.ibm.crail.hdfs.CrailHadoopFileSystem.java

License:Apache License

@Override
public boolean mkdirs(Path path, FsPermission permission) throws IOException {
    try {/*from ww w  . j av  a 2s.  c o m*/
        CrailDirectory file = dfs.create(path.toUri().getRawPath(), CrailNodeType.DIRECTORY,
                CrailStorageClass.PARENT, CrailLocationClass.DEFAULT).get().asDirectory();
        file.syncDir();
        return true;
    } catch (Exception e) {
        if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
            Path parent = path.getParent();
            mkdirs(parent);
            return mkdirs(path);
        } else if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_FILE_EXISTS])) {
            return true;
        } else {
            throw new IOException(e);
        }
    }
}

From source file:com.ibm.crail.hdfs.CrailHDFS.java

License:Apache License

@Override
public FSDataOutputStream createInternal(Path path, EnumSet<CreateFlag> flag, FsPermission absolutePermission,
        int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt,
        boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException,
        ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException {
    CrailFile fileInfo = null;/*from   w  w  w.jav a  2s. c  o  m*/
    try {
        fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT,
                CrailLocationClass.PARENT).get().asFile();
    } catch (Exception e) {
        if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
            fileInfo = null;
        } else {
            throw new IOException(e);
        }
    }

    if (fileInfo == null) {
        Path parent = path.getParent();
        this.mkdir(parent, FsPermission.getDirDefault(), true);
        try {
            fileInfo = dfs.create(path.toUri().getRawPath(), CrailNodeType.DATAFILE, CrailStorageClass.PARENT,
                    CrailLocationClass.PARENT).get().asFile();
        } catch (Exception e) {
            throw new IOException(e);
        }
    }

    CrailBufferedOutputStream outputStream = null;
    if (fileInfo != null) {
        try {
            fileInfo.syncDir();
            outputStream = fileInfo.getBufferedOutputStream(Integer.MAX_VALUE);
        } catch (Exception e) {
            throw new IOException(e);
        }
    } else {
        throw new IOException("Failed to create file, path " + path.toString());
    }

    if (outputStream != null) {
        return new CrailHDFSOutputStream(outputStream, statistics);
    } else {
        throw new IOException("Failed to create file, path " + path.toString());
    }
}

From source file:com.ibm.crail.hdfs.CrailHDFS.java

License:Apache License

@Override
public void mkdir(Path path, FsPermission permission, boolean createParent) throws AccessControlException,
        FileAlreadyExistsException, FileNotFoundException, UnresolvedLinkException, IOException {
    try {// w ww. j a  va 2  s  .  co m
        CrailDirectory file = dfs.create(path.toUri().getRawPath(), CrailNodeType.DIRECTORY,
                CrailStorageClass.PARENT, CrailLocationClass.DEFAULT).get().asDirectory();
        file.syncDir();
    } catch (Exception e) {
        if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_PARENT_MISSING])) {
            Path parent = path.getParent();
            mkdir(parent, permission, createParent);
            mkdir(path, permission, createParent);
        } else if (e.getMessage().contains(RpcErrors.messages[RpcErrors.ERR_FILE_EXISTS])) {
        } else {
            throw new IOException(e);
        }
    }
}