Example usage for org.apache.hadoop.fs Path getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException 

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static Path getTransactionInfoDir(Configuration conf) throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }/* ww  w.  j a v  a  2 s  .  c  o  m*/
    String working = conf.get(KEY_SYSTEM_DIR, DEFAULT_SYSTEM_DIR);
    Path path = new Path(working, TRANSACTION_INFO_DIR);
    return path.getFileSystem(conf).makeQualified(path);
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtilTest.java

License:Apache License

/**
 * Test for transaction info.//from w ww.jav a2 s.  c  o  m
 * @throws Exception if failed
 */
@Test
public void transactionInfo() throws Exception {
    Configuration conf = new Configuration();
    conf.set(HadoopDataSourceUtil.KEY_SYSTEM_DIR, folder.getRoot().getAbsoluteFile().toURI().toString());

    assertThat("empty system dir", folder.getRoot().listFiles(), is(new File[0]));
    assertThat(HadoopDataSourceUtil.findAllTransactionInfoFiles(conf).size(), is(0));

    Path t1 = HadoopDataSourceUtil.getTransactionInfoPath(conf, "ex1");
    assertThat(HadoopDataSourceUtil.getTransactionInfoExecutionId(t1), is("ex1"));
    t1.getFileSystem(conf).create(t1).close();

    assertThat(folder.getRoot().listFiles().length, is(greaterThan(0)));

    Path t2 = HadoopDataSourceUtil.getTransactionInfoPath(conf, "ex2");
    assertThat(t2, is(not(t1)));
    assertThat(HadoopDataSourceUtil.getTransactionInfoExecutionId(t2), is("ex2"));
    t2.getFileSystem(conf).create(t2).close();

    Path c2 = HadoopDataSourceUtil.getCommitMarkPath(conf, "ex2");
    assertThat(c2, is(not(t2)));
    c2.getFileSystem(conf).create(c2).close();

    List<Path> paths = new ArrayList<>();
    for (FileStatus stat : HadoopDataSourceUtil.findAllTransactionInfoFiles(conf)) {
        paths.add(stat.getPath());
    }
    assertThat(paths.size(), is(2));
    assertThat(paths, hasItem(t1));
    assertThat(paths, hasItem(t2));
}

---

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormat.java

License:Apache License

private List<InputSplit> getSplits(Configuration configuration, List<Path> paths) throws IOException {
    long splitSize = configuration.getLong(KEY_DEFAULT_SPLIT_SIZE, DEFAULT_SPLIT_SIZE);
    List<InputSplit> results = new ArrayList<>();
    for (Path path : paths) {
        FileSystem fs = path.getFileSystem(configuration);
        FileStatus[] statuses = fs.globStatus(path);
        if (statuses == null) {
            continue;
        }//from  w  w w .ja  v  a 2s  .c o m
        for (FileStatus status : statuses) {
            BlockMap blockMap = BlockMap.create(status.getPath().toString(), status.getLen(),
                    BlockMap.computeBlocks(fs, status), false);
            results.addAll(computeSplits(status.getPath(), blockMap, splitSize));
        }
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormatTest.java

License:Apache License

private FileStatus write(Configuration conf, int count) throws IOException {
    Path path = new Path(folder.newFile().toURI());
    try (ModelOutput<Text> output = TemporaryStorage.openOutput(conf, Text.class, path)) {
        Text buffer = new Text("Hello, world!");
        for (int i = 0; i < count; i++) {
            output.write(buffer);/* w w w  . jav  a2  s.  co m*/
        }
    }
    return path.getFileSystem(conf).getFileStatus(path);
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private List<Path> consumeLibraryPaths(LinkedList<String> rest) throws IOException {
    List<String> names = consumeLibraryNames(rest);
    if (names.isEmpty()) {
        return Collections.emptyList();
    }//ww  w  .  j a va2 s.  com
    List<Path> results = new ArrayList<>();
    LocalFileSystem local = FileSystem.getLocal(configuration);
    for (String name : names) {
        Path path = new Path(name);
        FileSystem fs;
        if (path.toUri().getScheme() == null) {
            fs = local;
        } else {
            fs = path.getFileSystem(configuration);
        }
        path = fs.makeQualified(path);
        if (fs.exists(path) == false) {
            throw new FileNotFoundException(path.toString());
        }
        results.add(path);
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private void configureJobJar(List<Path> paths, String className, Map<Path, Path> cacheMap) throws IOException {
    if (configuration.get(KEY_CONF_JAR) != null) {
        return;//from w w w  . ja  v  a  2  s.  c  o  m
    }
    for (Path path : paths) {
        Path remote = cacheMap.get(path);
        URI uri = path.toUri();
        if (remote != null && uri.getScheme().equals("file")) { //$NON-NLS-1$
            File file = new File(uri);
            if (isInclude(file, className)) {
                Path qualified = remote.getFileSystem(configuration).makeQualified(remote);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("Application class is in: file={2} ({1}), class={0}", //$NON-NLS-1$
                            className, path, qualified));
                }
                URI target = qualified.toUri();
                if (target.getScheme() != null
                        && (target.getScheme().equals("file") || target.getAuthority() != null)) { //$NON-NLS-1$
                    configuration.set(KEY_CONF_JAR, qualified.toString());
                }
                break;
            }
        }
    }
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private String buildLibjars(List<Path> paths) throws IOException {
    StringBuilder buf = new StringBuilder(configuration.get(KEY_CONF_LIBRARIES, "")); //$NON-NLS-1$
    for (Path path : paths) {
        if (buf.length() != 0) {
            buf.append(',');
        }//from  w ww  .jav a  2 s  .  c o m
        FileSystem fs = path.getFileSystem(configuration);
        buf.append(fs.makeQualified(path).toString());
    }
    String libjars = buf.toString();
    return libjars;
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private Path computeRepositoryPath() throws IOException {
    assert configuration.get(KEY_CACHE_REPOSITORY) != null;
    Path repositoryPath = new Path(configuration.get(KEY_CACHE_REPOSITORY));
    repositoryPath = repositoryPath.getFileSystem(configuration).makeQualified(repositoryPath);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Using cache repository: -D{0}={1}", //$NON-NLS-1$
                KEY_CACHE_REPOSITORY, repositoryPath));
    }//  w  w w .  j  a v a2 s  . c  o  m
    return repositoryPath;
}

---

From source file:com.asakusafw.runtime.stage.output.TemporaryOutputFormat.java

License:Apache License

@Override
public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }/*from   w  w  w  . jav a  2  s  .  com*/
    Path path = getOutputPath(context);
    if (TemporaryOutputFormat.getOutputPath(context) == null) {
        throw new IOException("Temporary output path is not set");
    }
    TokenCache.obtainTokensForNamenodes(context.getCredentials(), new Path[] { path },
            context.getConfiguration());
    if (path.getFileSystem(context.getConfiguration()).exists(path)) {
        throw new IOException(MessageFormat.format("Output directory {0} already exists", path));
    }
}

---

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

private Path findCacheForLocalMode(String resourceName, String localName) throws IOException {
    assert resourceName != null;
    assert localName != null;
    Path remotePath = null;
    String remoteName = null;//from  w ww.j av  a2  s  .c  o m
    for (URI uri : DistributedCache.getCacheFiles(configuration)) {
        if (localName.equals(uri.getFragment())) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("fragment matched: " + uri); //$NON-NLS-1$
            }
            String rpath = uri.getPath();
            remotePath = new Path(uri);
            remoteName = rpath.substring(rpath.lastIndexOf('/') + 1);
            break;
        }
    }
    if (remoteName == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("fragment not matched: " + resourceName); //$NON-NLS-1$
        }
        return null;
    }
    assert remotePath != null;
    for (Path path : getLocalCacheFiles()) {
        String localFileName = path.getName();
        if (remoteName.equals(localFileName) == false) {
            continue;
        }
        if (localFileSystem.exists(path) == false) {
            continue;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("local path matched: " + path); //$NON-NLS-1$
        }
        return localFileSystem.makeQualified(path);
    }
    FileSystem remoteFileSystem = remotePath.getFileSystem(configuration);
    remotePath = remoteFileSystem.makeQualified(remotePath);
    if (LOG.isDebugEnabled()) {
        LOG.debug("distributed cache is not localized explicitly: " + remotePath); //$NON-NLS-1$
    }
    if (isLocal(remoteFileSystem) == false) {
        LOG.warn(MessageFormat.format("Failed to resolve stage resource in local cache \"{1}\" (resource={0})",
                resourceName, localName));
    }
    return remotePath;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »