Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName() 

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.alibaba.jstorm.hdfs.common.rotation.MoveFileAction.java

License:Apache License

@Override
public void execute(FileSystem fileSystem, Path filePath) throws IOException {
    Path destPath = new Path(destination, filePath.getName());
    LOG.info("Moving file {} to {}", filePath, destPath);
    boolean success = fileSystem.rename(filePath, destPath);
    return;//from w w  w.j a  v a 2  s.c o m
}

---

From source file:com.alibaba.jstorm.hdfs.spout.FileLock.java

License:Apache License

/** returns lock on file or null if file is already locked. throws if unexpected problem */
public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, String spoutId)
        throws IOException {
    Path lockFile = new Path(lockDirPath, fileToLock.getName());

    try {//from   ww w  . j a va  2s. c  om
        FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
        if (ostream != null) {
            LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, lockFile, spoutId);
            return new FileLock(fs, lockFile, ostream, spoutId);
        } else {
            LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, spoutId);
            return null;
        }
    } catch (IOException e) {
        LOG.error("Error when acquiring lock on file " + fileToLock + " Spout = " + spoutId, e);
        throw e;
    }
}

---

From source file:com.alibaba.jstorm.hdfs.spout.FileLock.java

License:Apache License

/**
 * Finds a oldest expired lock file (using modification timestamp), then takes
 * ownership of the lock file/* w ww.java  2  s.  c o m*/
 * Impt: Assumes access to lockFilesDir has been externally synchronized such that
 *       only one thread accessing the same thread
 * @param fs
 * @param lockFilesDir
 * @param locktimeoutSec
 * @return
 */
public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec,
        String spoutId) throws IOException {
    // list files
    long now = System.currentTimeMillis();
    long olderThan = now - (locktimeoutSec * 1000);
    Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);

    // locate expired lock files (if any). Try to take ownership (oldest lock first)
    for (Path file : listing) {
        if (file.getName().equalsIgnoreCase(DirLock.DIR_LOCK_FILE)) {
            continue;
        }
        LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan);
        if (lastEntry != null) {
            FileLock lock = FileLock.takeOwnership(fs, file, lastEntry, spoutId);
            if (lock != null) {
                return lock;
            }
        }
    }
    if (listing.isEmpty()) {
        LOG.debug("No abandoned lock files found by Spout {}", spoutId);
    }
    return null;
}

---

From source file:com.alibaba.jstorm.hdfs.spout.FileLock.java

License:Apache License

/**
 * Finds oldest expired lock file (using modification timestamp), then takes
 * ownership of the lock file//from  w  ww.  j  a v  a 2  s  . c  om
 * Impt: Assumes access to lockFilesDir has been externally synchronized such that
 *       only one thread accessing the same thread
 * @param fs
 * @param lockFilesDir
 * @param locktimeoutSec
 * @return a Pair<lock file path, last entry in lock file> .. if expired lock file found
 * @throws IOException
 */
public static HdfsUtils.Pair<Path, LogEntry> locateOldestExpiredLock(FileSystem fs, Path lockFilesDir,
        int locktimeoutSec) throws IOException {
    // list files
    long now = System.currentTimeMillis();
    long olderThan = now - (locktimeoutSec * 1000);
    Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);

    // locate oldest expired lock file (if any) and take ownership
    for (Path file : listing) {
        if (file.getName().equalsIgnoreCase(DirLock.DIR_LOCK_FILE)) {
            continue;
        }
        LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan);
        if (lastEntry != null) {
            return new HdfsUtils.Pair<>(file, lastEntry);
        }
    }
    LOG.debug("No abandoned files found");
    return null;
}

---

From source file:com.alibaba.jstorm.hdfs.spout.HdfsSpout.java

License:Apache License

private FileReader pickNextFile() {
    try {// w  w  w.  j a v a 2s  . com
        // 1) If there are any abandoned files, pick oldest one
        lock = getOldestExpiredLock();
        if (lock != null) {
            LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile());
            Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath);
            String resumeFromOffset = lock.getLastLogEntry().fileOffset;
            LOG.info("Resuming processing of abandoned file : {}", file);
            return createFileReader(file, resumeFromOffset);
        }

        // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename it
        Collection<Path> listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath, 0);

        for (Path file : listing) {
            if (file.getName().endsWith(inprogress_suffix)) {
                continue;
            }
            if (file.getName().endsWith(ignoreSuffix)) {
                continue;
            }
            lock = FileLock.tryLock(hdfs, file, lockDirPath, spoutId);
            if (lock == null) {
                LOG.debug("Unable to get FileLock for {}, so skipping it.", file);
                continue; // could not lock, so try another file.
            }
            try {
                Path newFile = renameToInProgressFile(file);
                FileReader result = createFileReader(newFile);
                LOG.info("Processing : {} ", file);
                return result;
            } catch (Exception e) {
                LOG.error("Skipping file " + file, e);
                releaseLockAndLog(lock, spoutId);
                continue;
            }
        }

        return null;
    } catch (IOException e) {
        LOG.error("Unable to select next file for consumption " + sourceDirPath, e);
        return null;
    }
}

---

From source file:com.alibaba.jstorm.hdfs.spout.HdfsSpout.java

License:Apache License

/** Returns the corresponding input file in the 'sourceDirPath' for the specified lock file.
 *  If no such file is found then returns null
 *///from   w w w  .ja va  2  s  .c o m
private Path getFileForLockFile(Path lockFile, Path sourceDirPath) throws IOException {
    String lockFileName = lockFile.getName();
    Path dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName + inprogress_suffix);
    if (hdfs.exists(dataFile)) {
        return dataFile;
    }
    dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName);
    if (hdfs.exists(dataFile)) {
        return dataFile;
    }
    return null;
}

---

From source file:com.aliyun.fs.oss.blk.OssFileSystem.java

License:Apache License

@Override
public boolean rename(Path src, Path dst) throws IOException {
    Path absoluteSrc = makeAbsolute(src);
    INode srcINode = store.retrieveINode(absoluteSrc);
    if (srcINode == null) {
        // src path doesn't exist
        return false;
    }//from  w  w w . j  av a 2 s .  c  o  m
    Path absoluteDst = makeAbsolute(dst);
    INode dstINode = store.retrieveINode(absoluteDst);
    if (dstINode != null && dstINode.isDirectory()) {
        absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
        dstINode = store.retrieveINode(absoluteDst);
    }
    if (dstINode != null) {
        // dst path already exists - can't overwrite
        return false;
    }
    Path dstParent = absoluteDst.getParent();
    if (dstParent != null) {
        INode dstParentINode = store.retrieveINode(dstParent);
        if (dstParentINode == null || dstParentINode.isFile()) {
            // dst parent doesn't exist or is a file
            return false;
        }
    }
    return renameRecursive(absoluteSrc, absoluteDst);
}

---

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

@Override
public boolean rename(Path src, Path dst) throws IOException {

    String srcKey = pathToKey(makeAbsolute(src));

    if (srcKey.length() == 0) {
        // Cannot rename root of file system
        return false;
    }/*w w w  . j  a  v  a  2 s .c o m*/

    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";

    // Figure out the final destination
    String dstKey;
    try {
        boolean dstIsFile = !getFileStatus(dst).isDir();
        if (dstIsFile) {
            LOG.debug(debugPreamble + "returning false as dst is an already " + "existing file");
            // If dst is not a directory
            throw new FileAlreadyExistsException(
                    String.format("Failed to rename %s to %s, file already exists!", src, dst));
        } else {
            LOG.debug(debugPreamble + "using dst as output directory");
            dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
        }
    } catch (FileNotFoundException e) {
        LOG.debug(debugPreamble + "using dst as output destination");
        dstKey = pathToKey(makeAbsolute(dst));
        try {
            if (!getFileStatus(dst.getParent()).isDir()) {
                LOG.debug(debugPreamble + "returning false as dst parent exists and " + "is a file");
                return false;
            }
        } catch (FileNotFoundException ex) {
            LOG.debug(debugPreamble + "returning false as dst parent does not exist");
            throw ex;
        }
    }

    boolean srcIsFile;
    try {
        srcIsFile = !getFileStatus(src).isDir();
    } catch (FileNotFoundException e) {
        LOG.debug(debugPreamble + "returning false as src does not exist");
        throw e;
    }
    if (srcIsFile) {
        LOG.debug(debugPreamble + "src is file, so doing copy then delete in Oss");
        store.copy(srcKey, dstKey);
        store.delete(srcKey);
    } else {
        LOG.debug(debugPreamble + "src is directory, so copying contents");
        store.storeEmptyFile(dstKey + PATH_DELIMITER);

        List<String> keysToDelete = new ArrayList<String>();
        String priorLastKey = null;
        do {
            PartialListing listing = store.list(srcKey, OSS_MAX_LISTING_LENGTH, priorLastKey, true);
            for (FileMetadata file : listing.getFiles()) {
                keysToDelete.add(file.getKey());
                store.copy(file.getKey(), dstKey + file.getKey().substring(srcKey.length()));
            }
            priorLastKey = listing.getPriorLastKey();
        } while (priorLastKey != null);

        LOG.debug(debugPreamble + "all files in src copied, now removing " + "src files");
        for (String key : keysToDelete) {
            store.delete(key);
        }

        try {
            store.delete(srcKey + FOLDER_SUFFIX);
        } catch (FileNotFoundException e) {
            //this is fine, we don't require a marker
        }
        LOG.debug(debugPreamble + "done");
    }

    return true;
}

---

From source file:com.aliyun.odps.fs.VolumeFileSystem.java

License:Apache License

@Override
public boolean rename(Path src, Path dst) throws IOException {
    statistics.incrementWriteOps(1);/*w  w  w  .jav  a  2s.c  o  m*/
    Path absSrc = fixRelativePart(src);
    Path absDst = fixRelativePart(dst);
    if (!exists(absSrc)) {
        throw new FileNotFoundException("Source path " + src + " does not exist");
    }
    if (isDirectory(absDst)) {
        // destination is a directory: rename goes underneath it with the
        // source name
        absDst = new Path(absDst, absSrc.getName());
    }
    if (exists(absDst)) {
        throw new FileAlreadyExistsException("Destination path " + dst + " already exists");
    }
    if (absDst.getParent() != null && !exists(absDst.getParent())) {
        throw new FileNotFoundException(
                VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(absDst.getParent().toString()));
    }

    if (VolumeFSUtil.isParentOf(absSrc, absDst)) {
        throw new IOException("Cannot rename " + absSrc + " under itself" + " : " + absDst);
    }
    String srcPath = getPathName(absSrc);
    String dstPath = getPathName(absDst);
    try {
        return volumeClient.rename(srcPath, dstPath);
    } catch (VolumeException e) {
        logException(e);
        throw wrapExceptions(srcPath, e);
    }
}

---

From source file:com.architecting.ch07.MapReduceIndexerTool.java

License:Apache License

/** API for Java clients;visible for testing;may become a public API eventually */
int run(Options options) throws Exception {
    if (getConf().getBoolean("isMR1", false) && "local".equals(getConf().get("mapred.job.tracker"))) {
        throw new IllegalStateException(
                "Running with LocalJobRunner (i.e. all of Hadoop inside a single JVM) is not supported "
                        + "because LocalJobRunner does not (yet) implement the Hadoop Distributed Cache feature, "
                        + "which is required for passing files via --files and --libjars");
    }//from w  w w  .j  ava2  s.c om

    long programStartTime = System.nanoTime();
    getConf().setInt(SolrOutputFormat.SOLR_RECORD_WRITER_MAX_SEGMENTS, options.maxSegments);

    // switch off a false warning about allegedly not implementing Tool
    // also see http://hadoop.6.n7.nabble.com/GenericOptionsParser-warning-td8103.html
    // also see https://issues.apache.org/jira/browse/HADOOP-8183
    getConf().setBoolean("mapred.used.genericoptionsparser", true);

    if (options.log4jConfigFile != null) {
        Utils.setLogConfigFile(options.log4jConfigFile, getConf());
        addDistributedCacheFile(options.log4jConfigFile, getConf());
    }

    Configuration config = HBaseConfiguration.create();
    Job job = Job.getInstance(config);
    job.setJarByClass(getClass());

    // To be able to run this example from eclipse, we need to make sure 
    // the built jar is distributed to the map-reduce tasks from the
    // local file system.
    job.addCacheArchive(new URI("file:///home/cloudera/ahae/target/ahae.jar"));

    FileSystem fs = options.outputDir.getFileSystem(job.getConfiguration());
    if (fs.exists(options.outputDir) && !delete(options.outputDir, true, fs)) {
        return -1;
    }
    Path outputResultsDir = new Path(options.outputDir, RESULTS_DIR);
    Path outputReduceDir = new Path(options.outputDir, "reducers");

    int reducers = 1;

    Scan scan = new Scan();
    scan.addFamily(CF);
    // tag::SETUP[]
    scan.setCaching(500); // <1>
    scan.setCacheBlocks(false); // <2>

    TableMapReduceUtil.initTableMapperJob( // <3>
            options.inputTable, // Input HBase table name
            scan, // Scan instance to control what to index
            HBaseAvroToSOLRMapper.class, // Mapper to parse cells content.
            Text.class, // Mapper output key
            SolrInputDocumentWritable.class, // Mapper output value
            job);

    FileOutputFormat.setOutputPath(job, outputReduceDir);

    job.setJobName(getClass().getName() + "/" + Utils.getShortClassName(HBaseAvroToSOLRMapper.class));
    job.setReducerClass(SolrReducer.class); // <4>
    job.setPartitionerClass(SolrCloudPartitioner.class); // <5>
    job.getConfiguration().set(SolrCloudPartitioner.ZKHOST, options.zkHost);
    job.getConfiguration().set(SolrCloudPartitioner.COLLECTION, options.collection);
    job.getConfiguration().setInt(SolrCloudPartitioner.SHARDS, options.shards);

    job.setOutputFormatClass(SolrOutputFormat.class);
    SolrOutputFormat.setupSolrHomeCache(options.solrHomeDir, job);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(SolrInputDocumentWritable.class);
    job.setSpeculativeExecution(false);
    // end::SETUP[]
    job.setNumReduceTasks(reducers); // Set the number of reducers based on the number of shards we have.
    if (!waitForCompletion(job, true)) {
        return -1;// job failed
    }

    // -------------------------------------------------------------------------------------------------------------------------------------

    assert reducers == options.shards;

    // normalize output shard dir prefix, i.e.
    // rename part-r-00000 to part-00000 (stems from zero tree merge iterations)
    // rename part-m-00000 to part-00000 (stems from > 0 tree merge iterations)
    for (FileStatus stats : fs.listStatus(outputReduceDir)) {
        String dirPrefix = SolrOutputFormat.getOutputName(job);
        Path srcPath = stats.getPath();
        if (stats.isDirectory() && srcPath.getName().startsWith(dirPrefix)) {
            String dstName = dirPrefix + srcPath.getName().substring(dirPrefix.length() + "-m".length());
            Path dstPath = new Path(srcPath.getParent(), dstName);
            if (!rename(srcPath, dstPath, fs)) {
                return -1;
            }
        }
    }
    ;

    // publish results dir
    if (!rename(outputReduceDir, outputResultsDir, fs)) {
        return -1;
    }

    if (options.goLive && !new GoLive().goLive(options, listSortedOutputShardDirs(job, outputResultsDir, fs))) {
        return -1;
    }

    goodbye(job, programStartTime);
    return 0;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »