Example usage for org.apache.hadoop.fs Path isAbsolute

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path isAbsolute.

Prototype

public boolean isAbsolute() 

Source Link

Document

Returns true if the path component (i.e.

Usage

From source file:WikipediaForwardIndexBuilder.java

License:Apache License

@SuppressWarnings("static-access")
@Override/*from ww  w  . ja v a2s  .  c om*/
public int run(String[] args) throws Exception {
    Options options = new Options();
    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input").create(INPUT_OPTION));
    options.addOption(
            OptionBuilder.withArgName("path").hasArg().withDescription("index file").create(INDEX_FILE_OPTION));
    options.addOption(OptionBuilder.withArgName("en|sv|de|cs|es|zh|ar|tr").hasArg()
            .withDescription("two-letter language code").create(LANGUAGE_OPTION));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        return -1;
    }

    if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(INDEX_FILE_OPTION)) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(this.getClass().getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        return -1;
    }

    Path inputPath = new Path(cmdline.getOptionValue(INPUT_OPTION));
    String indexFile = cmdline.getOptionValue(INDEX_FILE_OPTION);

    String tmpPath = "tmp-" + WikipediaForwardIndexBuilder.class.getSimpleName() + "-" + RANDOM.nextInt(10000);

    if (!inputPath.isAbsolute()) {
        System.err.println("Error: " + INPUT_OPTION + " must be an absolute path!");
        return -1;
    }

    String language = null;
    if (cmdline.hasOption(LANGUAGE_OPTION)) {
        language = cmdline.getOptionValue(LANGUAGE_OPTION);
        if (language.length() != 2) {
            System.err.println("Error: \"" + language + "\" unknown language!");
            return -1;
        }
    }

    JobConf conf = new JobConf(getConf(), WikipediaForwardIndexBuilder.class);
    FileSystem fs = FileSystem.get(conf);

    LOG.info("Tool name: " + this.getClass().getName());
    LOG.info(" - input path: " + inputPath);
    LOG.info(" - index file: " + indexFile);
    LOG.info(" - language: " + language);
    LOG.info("Note: This tool only works on block-compressed SequenceFiles!");

    conf.setJobName(String.format("BuildWikipediaForwardIndex[%s: %s, %s: %s, %s: %s]", INPUT_OPTION, inputPath,
            INDEX_FILE_OPTION, indexFile, LANGUAGE_OPTION, language));

    conf.setNumReduceTasks(1);

    FileInputFormat.setInputPaths(conf, inputPath);
    FileOutputFormat.setOutputPath(conf, new Path(tmpPath));
    FileOutputFormat.setCompressOutput(conf, false);

    if (language != null) {
        conf.set("wiki.language", language);
    }

    conf.setInputFormat(NoSplitSequenceFileInputFormat.class);
    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapRunnerClass(MyMapRunner.class);
    conf.setReducerClass(IdentityReducer.class);

    // Delete the output directory if it exists already.
    fs.delete(new Path(tmpPath), true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int blocks = (int) counters.getCounter(Blocks.Total);

    LOG.info("number of blocks: " + blocks);

    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(tmpPath + "/part-00000")));
    FSDataOutputStream out = fs.create(new Path(indexFile), true);

    out.writeUTF(edu.umd.cloud9.collection.wikipedia.WikipediaForwardIndex.class.getCanonicalName());
    out.writeUTF(inputPath.toString());
    out.writeInt(blocks);

    int cnt = 0;
    Text line = new Text();
    while (reader.readLine(line) > 0) {
        String[] arr = line.toString().split("\\s+");

        int docno = Integer.parseInt(arr[0]);
        int offset = Integer.parseInt(arr[1]);
        short fileno = Short.parseShort(arr[2]);

        out.writeInt(docno);
        out.writeInt(offset);
        out.writeShort(fileno);

        cnt++;

        if (cnt % 100000 == 0) {
            LOG.info(cnt + " blocks written");
        }
    }

    reader.close();
    out.close();

    if (cnt != blocks) {
        throw new RuntimeException("Error: mismatch in block count!");
    }

    // Clean up.
    fs.delete(new Path(tmpPath), true);

    return 0;
}

---

From source file:RawParascaleFileSystem.java

License:Apache License

/**
 * {@inheritDoc}//from w ww.  ja  v  a2s .  com
 */
@Override
public File pathToFile(final Path path) {
    // this has public modifier due to subclassing
    checkPath(path);
    final String mountPoint = getMountPoint();
    final File file;

    if (path.isAbsolute()) {
        final URI pathUri = path.makeQualified(this.getUri(), this.getWorkingDirectory()).toUri();
        final String[] parseAuthority = parseAuthority(pathUri);

        file = new File(mountPoint,
                String.format("/%s/%s/%s", parseAuthority[1], parseAuthority[0], pathUri.getPath()));

    } else {
        file = new File(mountPoint, String.format("/%s/%s/%s/%s", controlNode, virtualFs, getRawHomeDirectory(),
                path.toUri().getPath()));
    }
    return file;
}

---

From source file:RawParascaleFileSystem.java

License:Apache License

/**
 * Converts a relative path to its absolute representation. If the path
 * already is an absolute path it will not be converted.
 *
 * @param aPath relative path/*from  w  ww. jav  a2 s. co m*/
 *
 * @return absolute path
 */
Path makeAbsolute(final Path aPath) {
    if (aPath.isAbsolute()) {
        return aPath;
    }
    return new Path(workingDirectory, aPath);
}

---

From source file:alluxio.hadoop.AbstractFileSystem.java

License:Apache License

@Override
public void setWorkingDirectory(Path path) {
    LOG.info("setWorkingDirectory({})", path);
    if (path.isAbsolute()) {
        mWorkingDir = path;/*from w  ww  . j a  v a 2 s . com*/
    } else {
        mWorkingDir = new Path(mWorkingDir, path);
    }
}

---

From source file:cascading.tap.hive.HiveTableDescriptor.java

License:Open Source License

/**
 * Constructs a new HiveTableDescriptor object.
 *
 * @param databaseName     The database name.
 * @param tableName        The table name
 * @param columnNames      Names of the columns
 * @param columnTypes      Hive types of the columns
 * @param delimiter        The field delimiter of the Hive table
 * @param serializationLib Hive serialization library.
 */// w  w w  . ja  v  a 2  s.  c  o m
public HiveTableDescriptor(String databaseName, String tableName, String[] columnNames, String[] columnTypes,
        String[] partitionKeys, String delimiter, String serializationLib, Path location) {
    if (tableName == null || tableName.isEmpty())
        throw new IllegalArgumentException("tableName cannot be null or empty");
    if (databaseName == null || tableName.isEmpty())
        this.databaseName = HIVE_DEFAULT_DATABASE_NAME;
    else
        this.databaseName = databaseName.toLowerCase();
    this.tableName = tableName.toLowerCase();
    this.columnNames = columnNames;
    this.columnTypes = columnTypes;
    this.partitionKeys = partitionKeys;
    this.serializationLib = serializationLib;
    //Only set the delimiter if the serialization lib is Delimited.
    if (delimiter == null && this.serializationLib == HIVE_DEFAULT_SERIALIZATION_LIB_NAME)
        this.delimiter = HIVE_DEFAULT_DELIMITER;
    else
        this.delimiter = delimiter;
    if (isPartitioned())
        verifyPartitionKeys();
    if (columnNames.length == 0 || columnTypes.length == 0 || columnNames.length != columnTypes.length)
        throw new IllegalArgumentException(
                "columnNames and columnTypes cannot be empty and must have the same size");

    if (location != null) {
        if (!location.isAbsolute())
            throw new IllegalArgumentException("location must be a fully qualified absolute path");

        // Store as string since path is not serialisable
        this.location = location.toString();
    }
}

---

From source file:com.aliyun.fs.oss.blk.JetOssFileSystemStore.java

License:Apache License

public static String pathToKey(Path path) {
    if (path.isAbsolute()) {
        // OSS File Path can not start with "/", so we
        // need to scratch the first "/".
        String absolutePath = path.toUri().getPath();
        return absolutePath.substring(1);
    }/*w w  w.  j  ava2s .c om*/
    return path.toUri().getPath();
}

---

From source file:com.aliyun.fs.oss.common.InMemoryFileSystemStore.java

License:Apache License

private Path normalize(Path path) {
    if (!path.isAbsolute()) {
        throw new IllegalArgumentException("Path must be absolute: " + path);
    }/* w  w w  .ja v  a 2 s  .  co  m*/
    return new Path(path.toUri().getPath());
}

---

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

private static String pathToKey(Path path) {
    if (!path.isAbsolute()) {
        throw new IllegalArgumentException("Path must be absolute: " + path);
    }//w  w w .  java  2s  .co  m
    // OSS File Path can not start with "/", so we need to scratch the first "/".
    String absolutePath = path.toUri().getPath();
    return absolutePath.substring(1);
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

@SuppressWarnings("unchecked")
private static List<Path> createFileListRelative(Counter counter, FileSystem fs, Path source)
        throws IOException {
    assert counter != null;
    assert fs != null;
    assert source != null;
    assert source.isAbsolute();
    URI baseUri = source.toUri();
    FileStatus root;/*from  ww w. jav  a  2s  . co m*/
    try {
        root = fs.getFileStatus(source);
    } catch (FileNotFoundException e) {
        LOG.warn(MessageFormat.format("Source path is not found: {0} (May be already moved)", baseUri));
        return Collections.emptyList();
    }
    counter.add(1);
    List<FileStatus> all = recursiveStep(fs, Collections.singletonList(root));
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Source path contains {1} files/directories: {0}", //$NON-NLS-1$
                baseUri, all.size()));
    }
    List<Path> results = new ArrayList<>();
    for (FileStatus stat : all) {
        if (FileSystemCompatibility.isDirectory(stat)) {
            continue;
        }
        Path path = stat.getPath();
        URI uri = path.toUri();
        URI relative = baseUri.relativize(uri);
        if (relative.equals(uri) == false) {
            results.add(new Path(relative));
        } else {
            throw new IOException(MessageFormat.format("Failed to compute relative path: base={0}, target={1}",
                    baseUri, uri));
        }
        counter.add(1);
    }
    Collections.sort(results);
    return results;
}

---

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path doResolve(Path sourcePath) throws IOException, InterruptedException {
    assert sourcePath.isAbsolute();
    FileSystem fs = sourcePath.getFileSystem(configuration);
    if (fs.exists(sourcePath) == false) {
        throw new FileNotFoundException(sourcePath.toString());
    }//from ww w.j  a v a2  s  . co m
    long sourceChecksum = computeChecksum(fs, sourcePath);
    Path cachePath = computeCachePath(sourcePath);
    Path cacheChecksumPath = computeCacheChecksumPath(cachePath);

    IOException firstException = null;
    RetryObject retry = retryStrategy
            .newInstance(MessageFormat.format("preparing cache ({0} -> {1})", sourcePath, cachePath));
    do {
        try (LockObject<? super Path> lock = lockProvider.tryLock(cachePath)) {
            // TODO reduce lock scope?
            if (lock == null) {
                continue;
            }
            if (isCached(cachePath, cacheChecksumPath, sourceChecksum)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache hit: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                // just returns cached file
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache miss: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                updateCache(sourcePath, sourceChecksum, cachePath, cacheChecksumPath);
            }
            return cachePath;
        } catch (IOException e) {
            LOG.warn(MessageFormat.format("Failed to prepare cache: {0} -> {1}", sourcePath, cachePath), e);
            if (firstException == null) {
                firstException = e;
            }
        }
    } while (retry.waitForNextAttempt());
    if (firstException == null) {
        throw new IOException(MessageFormat.format("Failed to acquire a lock for remote cache file: {0} ({1})",
                sourcePath, cachePath));
    }
    throw firstException;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »