Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName()

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.linkedin.thirdeye.hadoop.push.SegmentPushPhase.java

License:Apache License

public void pushOneTarFile(FileSystem fs, Path path) throws Exception {
    String fileName = path.getName();
    if (!fileName.endsWith(".tar.gz")) {
        return;//ww w . j  av a2 s.  co  m
    }
    long length = fs.getFileStatus(path).getLen();
    for (String host : hosts) {
        InputStream inputStream = null;
        try {
            inputStream = fs.open(path);
            fileName = fileName.split(".tar")[0];
            if (fileName.lastIndexOf(ThirdEyeConstants.SEGMENT_JOINER) != -1) {
                segmentName = fileName.substring(0, fileName.lastIndexOf(ThirdEyeConstants.SEGMENT_JOINER));
            }
            LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, port);
            try {
                int responseCode = FileUploadUtils.sendSegmentFile(host, port, fileName, inputStream, length);
                LOGGER.info("Response code: {}", responseCode);

                if (uploadSuccess == true && responseCode != 200) {
                    uploadSuccess = false;
                }

            } catch (Exception e) {
                LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {}  *******", fileName,
                        host, port);
                LOGGER.error("Caught exception during upload", e);
                throw new RuntimeException("Got Error during send tar files to push hosts!");
            }
        } finally {
            inputStream.close();
        }
    }
}

From source file:com.linkedin.thirdeye.hadoop.util.ThirdeyeAvroUtils.java

License:Apache License

private static DataFileStream<GenericRecord> getAvroReader(Path avroFile) throws IOException {
    FileSystem fs = FileSystem.get(new Configuration());
    if (avroFile.getName().endsWith("gz")) {
        return new DataFileStream<GenericRecord>(new GZIPInputStream(fs.open(avroFile)),
                new GenericDatumReader<GenericRecord>());
    } else {/*from  w  w  w .ja v  a2  s .  co m*/
        return new DataFileStream<GenericRecord>(fs.open(avroFile), new GenericDatumReader<GenericRecord>());
    }
}

From source file:com.lithium.flow.filer.HdfsFiler.java

License:Apache License

@Nonnull
@Override//from  w w  w .  jav  a  2s . co m
public Record getRecord(@Nonnull String path) throws IOException {
    Path filePath = new Path(path);
    try {
        FileStatus status = fileSystem.getFileStatus(filePath);
        if (status != null) {
            File file = new File(path);
            return getRecordForStatus(status, file.getParent() == null ? "" : file.getParent());
        }
    } catch (FileNotFoundException e) {
        // catch this here to avoid calling fileSystem.exists() that does the same thing
    }
    return new Record(getUri(), filePath.getParent().toString(), filePath.getName(), 0, -1, false);
}

From source file:com.m6d.filecrush.crush.Crush.java

License:Apache License

/**
 * Returns the output from {@link CrushReducer}. Each reducer writes out a mapping of source files to crush output file.
 *//*from   w ww. j a  v a  2 s  .c  om*/
private List<FileStatus> getOutputMappings() throws IOException {
    try {
        FileStatus[] files = fs.listStatus(outDir, new PathFilter() {
            Matcher matcher = Pattern.compile("part-\\d+").matcher("dummy");

            @Override
            public boolean accept(Path path) {
                matcher.reset(path.getName());

                return matcher.matches();
            }
        });

        return asList(files);
    } catch (FileNotFoundException e) {
        return new LinkedList<FileStatus>();
    }
}

From source file:com.m6d.filecrush.crush.Crush.java

License:Apache License

/**
 * Renames the source file to the destination file, taking into consideration that compression codes can mangle file names. Also
 * ensures that the parent directory of the destination exists.
 *
 * @param src//ww w . j av  a  2  s.c  om
 *          The path to the file to copy.
 * @param destDir
 *          The dir to which the file must be copied
 * @param fileName
 *          The new name of the file or null to keep the original file name
 *
 * @throws IOException
 */
private void rename(Path src, Path destDir, String fileName) throws IOException {
    fs.mkdirs(destDir);

    if (null != codecExtension && !fs.exists(src)) {
        /*
         * Try mangling the name like a codec would and invoke rename. Let execoptions bubble up.
         */
        src = new Path(src + codecExtension);
    }

    Path dest;

    if (null == fileName) {
        dest = new Path(destDir, src.getName());
    } else {
        dest = new Path(destDir, fileName);
    }

    fs.rename(src, dest);
}

From source file:com.marklogic.contentpump.DocumentPathFilter.java

License:Apache License

@Override
public boolean accept(Path inPath) {
    String filename = inPath.getName();
    if (filename.matches(pattern) == true) {
        return true;
    }// w  w  w.  j a  va  2 s  .c  o  m
    // take care of the case when INPUT_FILE_PATH is a DIR
    try {
        FileStatus[] status = fs.globStatus(inPath);
        if (status == null) {
            throw new IOException("Path in input_file_path doesn't exist: " + inPath);
        }
        for (FileStatus s : status) {
            if (s.isDirectory()) {
                return true;
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return false;
}

From source file:com.marklogic.contentpump.ImportRecordReader.java

License:Apache License

@SuppressWarnings("unchecked")
protected void configFileNameAsCollection(Configuration conf, Path file) {
    if (file == null) {
        return;//from   w ww.j  a  va  2s.com
    }
    if (conf.getBoolean(CONF_OUTPUT_FILENAME_AS_COLLECTION, false)) {
        if (value instanceof ContentWithFileNameWritable) {
            ((ContentWithFileNameWritable<VALUEIN>) value).setFileName(file.getName());
        } else {
            Writable cvalue = new ContentWithFileNameWritable<VALUEIN>((VALUEIN) value, file.getName());
            value = (VALUEIN) cvalue;
        }
    }
}

From source file:com.marklogic.contentpump.ZipDelimitedJSONReader.java

License:Apache License

@Override
protected void configFileNameAsCollection(Configuration conf, Path file) {
    String collectionName = file.getName() + "_" + currZipEntry.getName();
    super.configFileNameAsCollection(conf, new Path(collectionName));
}

From source file:com.mongodb.hadoop.BSONFileOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(final TaskAttemptContext context) throws IOException {
    // Open data output stream

    Path outPath = getDefaultWorkFile(context, ".bson");
    LOG.info("output going into " + outPath);

    FileSystem fs = outPath.getFileSystem(context.getConfiguration());
    FSDataOutputStream outFile = fs.create(outPath);

    FSDataOutputStream splitFile = null;
    if (MongoConfigUtil.getBSONOutputBuildSplits(context.getConfiguration())) {
        Path splitPath = new Path(outPath.getParent(), "." + outPath.getName() + ".splits");
        splitFile = fs.create(splitPath);
    }/*from   w w w.j a  va2s.co m*/

    long splitSize = BSONSplitter.getSplitSize(context.getConfiguration(), null);
    return new BSONFileRecordWriter<K, V>(outFile, splitFile, splitSize);
}

From source file:com.mongodb.hadoop.BSONPathFilter.java

License:Apache License

public boolean accept(final Path path) {
    String pathName = path.getName().toLowerCase();
    boolean acceptable = pathName.endsWith(".bson") && !pathName.startsWith(".");
    LOG.info(path.toString() + " returning " + acceptable);
    return acceptable;
}