Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:com.asakusafw.runtime.directio.hadoop.HadoopFileFormatAdapter.java

License:Apache License

@Override
public ModelOutput<T> createOutput(Class<? extends T> dataType, FileSystem fileSystem, final Path path,
        Counter counter) throws IOException, InterruptedException {
    FSDataOutputStream stream = fileSystem.create(path);
    boolean succeed = false;
    try {//from  w ww. j av a2  s  .  c  o m
        CountOutputStream cstream;
        if (LOG.isDebugEnabled()) {
            cstream = new CountOutputStream(stream, counter) {
                @Override
                public void close() throws IOException {
                    LOG.debug(MessageFormat.format("Start closing output (file={0})", //$NON-NLS-1$
                            path));
                    super.close();
                    LOG.debug(MessageFormat.format("Finish closing output (file={0})", //$NON-NLS-1$
                            path));
                }
            };
        } else {
            cstream = new CountOutputStream(stream, counter);
        }
        ModelOutput<T> output = streamFormat.createOutput(dataType, path.toString(), cstream);
        succeed = true;
        return output;
    } finally {
        if (succeed == false) {
            try {
                stream.close();
            } catch (IOException e) {
                LOG.warn(MessageFormat.format("Failed to close output (path={0})", path), e);
            }
        }
    }
}

---

From source file:com.asakusafw.runtime.stage.AbstractCleanupStageClient.java

License:Apache License

@Override
protected int execute(String[] args) throws IOException, InterruptedException {
    Configuration conf = getConf();
    Path path = getPath(conf);
    FileSystem fileSystem = FileSystem.get(path.toUri(), conf);
    String info = MessageFormat.format("batchId={0}, flowId={1}, executionId={2}, operationId={3}, path={4}", //$NON-NLS-1$
            getBatchId(), getFlowId(), getExecutionId(), getOperationId(), path);
    try {/*from www .j a  va 2 s.  c o  m*/
        LOG.info(MessageFormat.format("Searching for cleanup target: {0}", info));
        long start = System.currentTimeMillis();
        if (RuntimeContext.get().isSimulation()) {
            LOG.info(MessageFormat.format(
                    "Skip deleting cleanup target because current execution is in simulation mode: {0}", info));
        } else {
            FileStatus stat = fileSystem.getFileStatus(path);
            if (stat == null) {
                throw new FileNotFoundException(path.toString());
            }
            LOG.info(MessageFormat.format("Start deleting cleanup target: {0}", info));
            if (fileSystem.delete(path, true) == false) {
                throw new IOException("FileSystem.delete() returned false");
            }
        }
        long end = System.currentTimeMillis();
        LOG.info(MessageFormat.format("Finish deleting cleanup target: {0}, elapsed={1}ms", info, end - start));
        return 0;
    } catch (FileNotFoundException e) {
        LOG.warn(MessageFormat.format("Cleanup target is missing: {0}", info));
        return 0;
    } catch (IOException e) {
        LOG.warn(MessageFormat.format("Failed to delete cleanup target: {0}", info), e);
        return 1;
    } finally {
        FileSystem.closeAll();
    }
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private List<Path> consumeLibraryPaths(LinkedList<String> rest) throws IOException {
    List<String> names = consumeLibraryNames(rest);
    if (names.isEmpty()) {
        return Collections.emptyList();
    }//from  ww w .j  a  v a  2 s  .  co m
    List<Path> results = new ArrayList<>();
    LocalFileSystem local = FileSystem.getLocal(configuration);
    for (String name : names) {
        Path path = new Path(name);
        FileSystem fs;
        if (path.toUri().getScheme() == null) {
            fs = local;
        } else {
            fs = path.getFileSystem(configuration);
        }
        path = fs.makeQualified(path);
        if (fs.exists(path) == false) {
            throw new FileNotFoundException(path.toString());
        }
        results.add(path);
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private void configureJobJar(List<Path> paths, String className, Map<Path, Path> cacheMap) throws IOException {
    if (configuration.get(KEY_CONF_JAR) != null) {
        return;/*from  ww  w  . j  a va  2 s  . c o  m*/
    }
    for (Path path : paths) {
        Path remote = cacheMap.get(path);
        URI uri = path.toUri();
        if (remote != null && uri.getScheme().equals("file")) { //$NON-NLS-1$
            File file = new File(uri);
            if (isInclude(file, className)) {
                Path qualified = remote.getFileSystem(configuration).makeQualified(remote);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("Application class is in: file={2} ({1}), class={0}", //$NON-NLS-1$
                            className, path, qualified));
                }
                URI target = qualified.toUri();
                if (target.getScheme() != null
                        && (target.getScheme().equals("file") || target.getAuthority() != null)) { //$NON-NLS-1$
                    configuration.set(KEY_CONF_JAR, qualified.toString());
                }
                break;
            }
        }
    }
}

---

From source file:com.asakusafw.runtime.stage.output.TemporaryOutputFormat.java

License:Apache License

/**
 * Configures output path./*from   www  .ja va  2  s .co  m*/
 * @param context current context
 * @param path target output path
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public static void setOutputPath(JobContext context, Path path) {
    if (context == null) {
        throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$
    }
    if (path == null) {
        throw new IllegalArgumentException("path must not be null"); //$NON-NLS-1$
    }
    context.getConfiguration().set(KEY_OUTPUT_PATH, path.toString());
}

---

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path doResolve(Path sourcePath) throws IOException, InterruptedException {
    assert sourcePath.isAbsolute();
    FileSystem fs = sourcePath.getFileSystem(configuration);
    if (fs.exists(sourcePath) == false) {
        throw new FileNotFoundException(sourcePath.toString());
    }//from w ww .j  a v  a 2s  .c o  m
    long sourceChecksum = computeChecksum(fs, sourcePath);
    Path cachePath = computeCachePath(sourcePath);
    Path cacheChecksumPath = computeCacheChecksumPath(cachePath);

    IOException firstException = null;
    RetryObject retry = retryStrategy
            .newInstance(MessageFormat.format("preparing cache ({0} -> {1})", sourcePath, cachePath));
    do {
        try (LockObject<? super Path> lock = lockProvider.tryLock(cachePath)) {
            // TODO reduce lock scope?
            if (lock == null) {
                continue;
            }
            if (isCached(cachePath, cacheChecksumPath, sourceChecksum)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache hit: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                // just returns cached file
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("cache miss: {0} -> {1}", //$NON-NLS-1$
                            sourcePath, cachePath));
                }
                updateCache(sourcePath, sourceChecksum, cachePath, cacheChecksumPath);
            }
            return cachePath;
        } catch (IOException e) {
            LOG.warn(MessageFormat.format("Failed to prepare cache: {0} -> {1}", sourcePath, cachePath), e);
            if (firstException == null) {
                firstException = e;
            }
        }
    } while (retry.waitForNextAttempt());
    if (firstException == null) {
        throw new IOException(MessageFormat.format("Failed to acquire a lock for remote cache file: {0} ({1})",
                sourcePath, cachePath));
    }
    throw firstException;
}

---

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

private Path computeCachePath(Path file) {
    assert repository != null;
    String directoryName;//from  w w w . j av  a 2s  .c  o m
    Path parent = file.getParent();
    if (parent == null) {
        directoryName = String.format("%08x", 0); //$NON-NLS-1$
    } else {
        directoryName = String.format("%08x", parent.toString().hashCode()); //$NON-NLS-1$
    }
    Path directory = new Path(repository, directoryName);
    Path target = new Path(directory, file.getName());
    return target;
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGetTest.java

License:Apache License

/**
 * Gets a single file.//from   w w  w  .  j a va2s  . c o  m
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    Path testing = new Path(PREFIX, "testing");
    put(testing, "Hello, world!");

    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    int result = new WindGateHadoopGet(conf).execute(buffer, testing.toString());
    assertThat(result, is(0));

    Map<String, String> contents = get(buffer.toByteArray());
    assertThat(contents.size(), is(1));
    assertThat(contents.get("testing"), is("Hello, world!"));
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGetTest.java

License:Apache License

/**
 * Gets multiple files.//from   www.  j  av a2s. c o m
 * @throws Exception if failed
 */
@Test
public void multiple() throws Exception {
    Path path1 = new Path(PREFIX, "testing-1");
    Path path2 = new Path(PREFIX, "testing-2");
    Path path3 = new Path(PREFIX, "testing-3");
    put(path1, "Hello1, world!");
    put(path2, "Hello2, world!");
    put(path3, "Hello3, world!");

    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    int result = new WindGateHadoopGet(conf).execute(buffer, path1.toString(), path2.toString(),
            path3.toString());
    assertThat(result, is(0));

    Map<String, String> contents = get(buffer.toByteArray());
    assertThat(contents.size(), is(3));
    assertThat(contents.get("testing-1"), is("Hello1, world!"));
    assertThat(contents.get("testing-2"), is("Hello2, world!"));
    assertThat(contents.get("testing-3"), is("Hello3, world!"));
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGetTest.java

License:Apache License

/**
 * Attemts to get missing files./* w  w w .j av  a2 s  .c o  m*/
 * @throws Exception if failed
 */
@Test
public void missing() throws Exception {
    Path testing = new Path(PREFIX, "testing");

    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    int result = new WindGateHadoopGet(conf).execute(buffer, testing.toString());
    assertThat(result, is(not(0)));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »