Example usage for org.apache.hadoop.fs Path getFileSystem

List of usage examples for org.apache.hadoop.fs Path getFileSystem

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java

License:Apache License

/**
 * Resolves the raw path pattern into the concrete file status list.
 * @param conf current configuration//from   ww  w .  j  a v a 2s  . com
 * @param pathPattern path pattern which describes temporary storage
 * @return the resolved file status
 * @throws IOException if failed to resolve path pattern
 * @throws IllegalArgumentException if some parameters were {@code null}
 * @since 0.7.1
 */
public static List<FileStatus> listStatus(Configuration conf, Path pathPattern) throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (pathPattern == null) {
        throw new IllegalArgumentException("pathPattern must not be null"); //$NON-NLS-1$
    }
    FileSystem fs = pathPattern.getFileSystem(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Listing temporary input: {0} (fs={1})", //$NON-NLS-1$
                pathPattern, fs.getUri()));
    }
    FileStatus[] statusList = fs.globStatus(pathPattern);
    if (statusList == null || statusList.length == 0) {
        return Collections.emptyList();
    }
    return Arrays.asList(statusList);
}

From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java

License:Apache License

/**
 * Opens a temporary {@link ModelInput} for the specified path.
 * @param <V> data type//from ww  w . j  a  v a  2s  . com
 * @param conf configuration
 * @param dataType data type
 * @param path source path (must not contain wildcards)
 * @return the opened {@link ModelInput}
 * @throws IOException if failed to open input
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
@SuppressWarnings("unchecked")
public static <V> ModelInput<V> openInput(Configuration conf, Class<V> dataType, Path path) throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (dataType == null) {
        throw new IllegalArgumentException("dataType must not be null"); //$NON-NLS-1$
    }
    if (path == null) {
        throw new IllegalArgumentException("path must not be null"); //$NON-NLS-1$
    }
    FileSystem fs = path.getFileSystem(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Opening temporary input: {0} (fs={1})", //$NON-NLS-1$
                path, fs.getUri()));
    }
    if (Writable.class.isAssignableFrom(dataType)) {
        return (ModelInput<V>) new TemporaryFileInput<>(fs.open(path), 0);
    }
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
    return (ModelInput<V>) new SequenceFileModelInput<>(reader);
}

From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java

License:Apache License

/**
 * Opens a temporary {@link ModelOutput} for the specified path.
 * @param <V> data type/*from  www .  ja  va2s .  com*/
 * @param conf configuration
 * @param dataType data type
 * @param path target path
 * @return the opened {@link ModelOutput}
 * @throws IOException if failed to open output
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
@SuppressWarnings("unchecked")
public static <V> ModelOutput<V> openOutput(Configuration conf, Class<V> dataType, Path path)
        throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (dataType == null) {
        throw new IllegalArgumentException("dataType must not be null"); //$NON-NLS-1$
    }
    if (path == null) {
        throw new IllegalArgumentException("path must not be null"); //$NON-NLS-1$
    }
    FileSystem fs = path.getFileSystem(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Opening temporary output: {0} (fs={1})", //$NON-NLS-1$
                path, fs.getUri()));
    }
    if (Writable.class.isAssignableFrom(dataType)) {
        return (ModelOutput<V>) new TemporaryFileOutput<>(fs.create(path, true), dataType.getName(),
                OUTPUT_INIT_BUFFER_SIZE, OUTPUT_PAGE_SIZE);
    }
    SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, path, NullWritable.class, dataType);
    return new SequenceFileModelOutput<>(out);
}

From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java

License:Apache License

/**
 * Opens a temporary {@link ModelOutput} for the specified path.
 * @param <V> data type/*www  .  j  av  a2 s. co m*/
 * @param conf configuration
 * @param dataType data type
 * @param path target path
 * @param compressionCodec compression codec, or null if not compressed
 * @return the opened {@link ModelOutput}
 * @throws IOException if failed to open output
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
@SuppressWarnings("unchecked")
public static <V> ModelOutput<V> openOutput(Configuration conf, Class<V> dataType, Path path,
        CompressionCodec compressionCodec) throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (dataType == null) {
        throw new IllegalArgumentException("dataType must not be null"); //$NON-NLS-1$
    }
    if (path == null) {
        throw new IllegalArgumentException("path must not be null"); //$NON-NLS-1$
    }
    FileSystem fs = path.getFileSystem(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Opening temporary output: {0} (fs={1})", //$NON-NLS-1$
                path, fs.getUri()));
    }
    if (Writable.class.isAssignableFrom(dataType)) {
        return (ModelOutput<V>) new TemporaryFileOutput<>(fs.create(path, true), dataType.getName(),
                OUTPUT_INIT_BUFFER_SIZE, OUTPUT_PAGE_SIZE);
    }
    SequenceFile.Writer out = newWriter(conf, fs, dataType, path, compressionCodec);
    return new SequenceFileModelOutput<>(out);
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

@Override
public Path resolve(Path file) throws IOException, InterruptedException {
    FileSystem fs = file.getFileSystem(configuration);
    Path qualified = fs.makeQualified(file);
    return doResolve(qualified);
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path doResolve(Path sourcePath) throws IOException, InterruptedException {
    assert sourcePath.isAbsolute();
    FileSystem fs = sourcePath.getFileSystem(configuration);
    if (fs.exists(sourcePath) == false) {
        throw new FileNotFoundException(sourcePath.toString());
    }/*from w  w w  .ja va  2  s .  c o m*/
    long sourceChecksum = computeChecksum(fs, sourcePath);
    Path cachePath = computeCachePath(sourcePath);
    Path cacheChecksumPath = computeCacheChecksumPath(cachePath);

    IOException firstException = null;
    RetryObject retry = retryStrategy
            .newInstance(MessageFormat.format("preparing cache ({0} -> {1})", sourcePath, cachePath));
    do {
        try (LockObject<? super Path> lock = lockProvider.tryLock(cachePath)) {
            // TODO reduce lock scope?
            if (lock == null) {
                continue;
            }
            if (isCached(cachePath, cacheChecksumPath, sourceChecksum)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache hit: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                // just returns cached file
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache miss: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                updateCache(sourcePath, sourceChecksum, cachePath, cacheChecksumPath);
            }
            return cachePath;
        } catch (IOException e) {
            LOG.warn(MessageFormat.format("Failed to prepare cache: {0} -> {1}", sourcePath, cachePath), e);
            if (firstException == null) {
                firstException = e;
            }
        }
    } while (retry.waitForNextAttempt());
    if (firstException == null) {
        throw new IOException(MessageFormat.format("Failed to acquire a lock for remote cache file: {0} ({1})",
                sourcePath, cachePath));
    }
    throw firstException;
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private boolean isCached(Path cacheFilePath, Path cacheChecksumPath, long checksum) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("checking remote cache: {0}", //$NON-NLS-1$
                cacheFilePath));//ww  w.  j av a 2s . c  o  m
    }
    FileSystem fs = cacheChecksumPath.getFileSystem(configuration);
    if (fs.exists(cacheChecksumPath) == false || fs.exists(cacheFilePath) == false) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("remote cache is not found: {0}", //$NON-NLS-1$
                    cacheFilePath));
        }
        return false;
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("reading remote cache checksum: {0}", //$NON-NLS-1$
                    cacheFilePath));
        }
        long other;
        try (FSDataInputStream input = fs.open(cacheChecksumPath)) {
            other = input.readLong();
        }
        return checksum == other;
    }
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private void updateCache(Path file, long checksum, Path cachePath, Path cacheChecksumPath) throws IOException {
    if (LOG.isInfoEnabled()) {
        LOG.info(MessageFormat.format("updating library cache: {0} -> {1}", file, cachePath));
    }//from   w w w . j  ava 2s  .c  o m

    FileSystem sourceFs = file.getFileSystem(configuration);
    FileSystem cacheFs = cachePath.getFileSystem(configuration);

    // remove checksum file -> cachePath
    delete(cacheFs, cacheChecksumPath);
    delete(cacheFs, cachePath);

    // sync source file to cache file
    try (FSDataOutputStream checksumOutput = cacheFs.create(cacheChecksumPath, false)) {
        checksumOutput.writeLong(checksum);
        syncFile(sourceFs, file, cacheFs, cachePath);
    }
}

From source file:com.asakusafw.testdriver.executor.DefaultDeleteTaskExecutor.java

License:Apache License

@Override
protected void deleteOnHadoopFileSystem(TaskExecutionContext context, String path) throws IOException {
    Configuration conf = context.findResource(Configuration.class).get();
    Path p = new Path(path);
    FileSystem fs = p.getFileSystem(conf);
    try {//from   www  .  java2s . c  o  m
        fs.delete(p, true);
    } catch (FileNotFoundException e) {
        // may not occur
        LOG.debug("unexpected exception was occurred", e);
    }
}

From source file:com.asakusafw.testdriver.file.FileDeployer.java

License:Apache License

private void copy(File result, String destination) throws IOException {
    assert result != null;
    assert destination != null;
    Path target = new Path(destination);
    FileSystem fs = target.getFileSystem(configuration);
    fs.copyFromLocalFile(new Path(result.toURI()), target);
}