Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName()

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.lightboxtechnologies.spectrum.ExtractDataMapper.java

License:Apache License

SequenceFile.Reader openExtentsFile(FileSystem hdpFs, Configuration conf) throws IOException {
    SequenceFile.Reader extents = null;

    final Path[] files = DistributedCache.getLocalCacheFiles(conf);
    if (files != null && files.length > 0) {
        String extentsname = conf.get("com.lbt.extentsname");
        final LocalFileSystem localfs = FileSystem.getLocal(conf);
        boolean found = false;
        for (Path p : files) {
            if (p.getName().equals(extentsname)) {
                found = true;//from w w  w .j  a v a 2s . c o m
                LOG.info("Opening extents file " + p);
                extents = new SequenceFile.Reader(localfs, p, conf);
                break;
            }
        }
        if (!found) {
            LOG.warn("Could not find extents file in local cache named " + extentsname);
        }
    } else if (files == null) {
        throw new RuntimeException("No file paths retrieved from distributed cache");
        // extents = new SequenceFile.Reader(hdpFs, new Path("ceic_extents/part-r-00000"), conf); // TO-DO: fix hard-coding
    }

    if (extents == null) {
        throw new RuntimeException(
                "Could not open extents file. Number of files in the cache: " + files.length);
    }

    return extents;
}

From source file:com.linkedin.cubert.plan.physical.JobExecutor.java

License:Open Source License

protected void setLibjars() throws IOException {
    if (!root.has("libjars"))
        return;/*from   w w w  .  j  av  a2s. c  o  m*/

    FileSystem localFs = FileSystem.getLocal(conf);

    for (JsonNode node : asArray(root, "libjars")) {
        Path path = new Path(node.getTextValue());

        if (localFs.exists(path)) {
            Path dstPath = new Path(tmpDir, path.getName());
            fs.copyFromLocalFile(path, dstPath);

            path = dstPath;
        }

        DistributedCache.addFileToClassPath(path, conf, fs);

    }
}

From source file:com.linkedin.cubert.plan.physical.JobExecutor.java

License:Open Source License

protected void cacheIndex() throws IOException, InstantiationException, IllegalAccessException,
        ClassNotFoundException, URISyntaxException {
    if (!root.has("cacheIndex"))
        return;//  w  ww.  j  a  v a2 s .  co m

    for (JsonNode indexNode : root.path("cacheIndex")) {
        // extract the index named by "index" from the location specified in "path";
        Index indexToCache = Index.extractFromRelation(conf, getText(indexNode, "path"));

        String indexName = JsonUtils.getText(indexNode, "name");

        Path indexPath = new Path(tmpDir, UUID.randomUUID().toString());
        SerializerUtils.serializeToFile(conf, indexPath, indexToCache);

        DistributedCache.addCacheFile(new URI(indexPath.toString() + "#" + indexName), conf);

        // tmpFiles.add(indexPath);

        conf.set(CubertStrings.JSON_CACHE_INDEX_PREFIX + indexName, indexPath.getName());

        print.f("Caching index at path [%s] as [%s]", getText(indexNode, "path"), indexPath.toString());
    }

}

From source file:com.linkedin.mapred.AvroHdfsFileReader.java

License:Open Source License

private boolean isAvro(Path path) {
    return path.getName().endsWith(".avro");
}

From source file:com.linkedin.mapred.AvroUtils.java

License:Open Source License

/**
 * Check if the path should be ignored. Currently only paths with "_log" are ignored.
 * // w w w . jav a  2 s.com
 * @param path
 * @return
 * @throws IOException
 */
public static boolean shouldPathBeIgnored(Path path) throws IOException {
    return path.getName().startsWith("_");
}

From source file:com.linkedin.mapred.AvroUtils.java

License:Open Source License

public static FileStatus[] getAvroPartFiles(JobConf conf, Path outPath) throws IOException {
    Path outputPath = outPath;/*from  w w w  .  j  a va2s.co  m*/
    FileSystem fileSystem = outputPath.getFileSystem(conf);

    FileStatus[] partFiles = fileSystem.listStatus(outputPath, new PathFilter() {
        @Override
        public boolean accept(Path path) {
            if (path.getName().endsWith(".avro")) {
                return true;
            }
            return false;
        }
    });

    return partFiles;
}

From source file:com.linkedin.mr_kluj.HiddenFilePathFilter.java

License:Apache License

public boolean accept(Path path) {
    String name = path.getName();
    return !name.startsWith("_") && !name.startsWith(".");
}

From source file:com.linkedin.pinot.hadoop.job.SegmentCreationJob.java

License:Apache License

private void addDepsJarToDistributedCache(Path path, Job job) throws IOException {
    LOGGER.info("Trying to add all the deps jar files from directory: {}", path);
    FileSystem fs = FileSystem.get(getConf());
    FileStatus[] fileStatusArr = fs.listStatus(path);
    for (FileStatus fileStatus : fileStatusArr) {
        if (fileStatus.isDirectory()) {
            addDepsJarToDistributedCache(fileStatus.getPath(), job);
        } else {/*from  w  w  w  .  j  av a 2  s . com*/
            Path depJarPath = fileStatus.getPath();
            if (depJarPath.getName().endsWith(".jar")) {
                LOGGER.info("Adding deps jar files: {}", path);
                job.addCacheArchive(path.toUri());
            }
        }
    }
}

From source file:com.linkedin.pinot.hadoop.job.SegmentTarPushJob.java

License:Apache License

public void pushOneTarFile(FileSystem fs, Path path) throws Exception {
    String fileName = path.getName();
    if (!fileName.endsWith(".tar.gz")) {
        return;// w w w.j ava  2 s  . c  o m
    }
    long length = fs.getFileStatus(path).getLen();
    for (String host : _hosts) {
        InputStream inputStream = null;
        try {
            inputStream = fs.open(path);
            fileName = fileName.split(".tar")[0];
            LOGGER.info("******** Upoading file: {} to Host: {} and Port: {} *******", fileName, host, _port);
            try {
                int responseCode = FileUploadUtils.sendSegmentFile(host, _port, fileName, inputStream, length);
                LOGGER.info("Response code: {}", responseCode);
            } catch (Exception e) {
                LOGGER.error("******** Error Upoading file: {} to Host: {} and Port: {}  *******", fileName,
                        host, _port);
                LOGGER.error("Caught exception during upload", e);
                throw new RuntimeException("Got Error during send tar files to push hosts!");
            }
        } finally {
            inputStream.close();
        }
    }
}

From source file:com.linkedin.pinot.hadoop.job.SegmentUriPushJob.java

License:Apache License

public void pushOneTarFile(FileSystem fs, Path path) throws Exception {
    String fileName = path.getName();
    if (!fileName.endsWith(".tar.gz")) {
        return;// ww  w . java  2  s .  com
    }
    for (String host : _hosts) {
        String uri = String.format("%s%s%s", _pushUriPrefix, path.toUri().getRawPath(), _pushUriSuffix);
        LOGGER.info("******** Upoading file: {} to Host: {} and Port: {} with download uri: {} *******",
                fileName, host, _port, uri);
        try {
            int responseCode = FileUploadUtils.sendSegmentUri(host, _port, uri);
            LOGGER.info("Response code: {}", responseCode);
        } catch (Exception e) {
            LOGGER.error("******** Error Upoading file: {} to Host: {} and Port: {}  *******", fileName, host,
                    _port);
            LOGGER.error("Caught exception during upload", e);
            throw new RuntimeException("Got Error during send tar files to push hosts!");
        }
    }
}