Example usage for org.apache.hadoop.fs Path toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri() 

Source Link

Document

Convert this Path to a URI.

Usage

From source file:com.asakusafw.runtime.stage.AbstractCleanupStageClient.java

License:Apache License

@Override
protected int execute(String[] args) throws IOException, InterruptedException {
    Configuration conf = getConf();
    Path path = getPath(conf);
    FileSystem fileSystem = FileSystem.get(path.toUri(), conf);
    String info = MessageFormat.format("batchId={0}, flowId={1}, executionId={2}, operationId={3}, path={4}", //$NON-NLS-1$
            getBatchId(), getFlowId(), getExecutionId(), getOperationId(), path);
    try {//from   w w w . ja  va2  s . com
        LOG.info(MessageFormat.format("Searching for cleanup target: {0}", info));
        long start = System.currentTimeMillis();
        if (RuntimeContext.get().isSimulation()) {
            LOG.info(MessageFormat.format(
                    "Skip deleting cleanup target because current execution is in simulation mode: {0}", info));
        } else {
            FileStatus stat = fileSystem.getFileStatus(path);
            if (stat == null) {
                throw new FileNotFoundException(path.toString());
            }
            LOG.info(MessageFormat.format("Start deleting cleanup target: {0}", info));
            if (fileSystem.delete(path, true) == false) {
                throw new IOException("FileSystem.delete() returned false");
            }
        }
        long end = System.currentTimeMillis();
        LOG.info(MessageFormat.format("Finish deleting cleanup target: {0}, elapsed={1}ms", info, end - start));
        return 0;
    } catch (FileNotFoundException e) {
        LOG.warn(MessageFormat.format("Cleanup target is missing: {0}", info));
        return 0;
    } catch (IOException e) {
        LOG.warn(MessageFormat.format("Failed to delete cleanup target: {0}", info), e);
        return 1;
    } finally {
        FileSystem.closeAll();
    }
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private List<Path> consumeLibraryPaths(LinkedList<String> rest) throws IOException {
    List<String> names = consumeLibraryNames(rest);
    if (names.isEmpty()) {
        return Collections.emptyList();
    }/*w  w w.  j  a  v  a 2 s . c o  m*/
    List<Path> results = new ArrayList<>();
    LocalFileSystem local = FileSystem.getLocal(configuration);
    for (String name : names) {
        Path path = new Path(name);
        FileSystem fs;
        if (path.toUri().getScheme() == null) {
            fs = local;
        } else {
            fs = path.getFileSystem(configuration);
        }
        path = fs.makeQualified(path);
        if (fs.exists(path) == false) {
            throw new FileNotFoundException(path.toString());
        }
        results.add(path);
    }
    return results;
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private List<URL> processLibraries(List<Path> libraryPaths, String applicationClassName)
        throws IOException, InterruptedException {
    if (libraryPaths.isEmpty()) {
        return Collections.emptyList();
    }//from w ww .ja v a2 s.  c om
    Map<Path, Path> resolved = processLibraryCache(libraryPaths);
    List<URL> localUrls = new ArrayList<>();
    List<Path> remotePaths = new ArrayList<>();
    for (Path path : libraryPaths) {
        URI uri = path.toUri();
        assert uri.getScheme() != null;
        if (uri.getScheme().equals("file")) { //$NON-NLS-1$
            localUrls.add(uri.toURL());
        }
        Path remote = resolved.get(path);
        if (remote == null) {
            remotePaths.add(path);
        } else {
            remotePaths.add(remote);
        }
    }

    if (configuration.getBoolean(KEY_CACHE_JOBJAR, DEFAULT_CACHE_JOBJAR)) {
        configureJobJar(libraryPaths, applicationClassName, resolved);
    }

    String libjars = buildLibjars(remotePaths);
    if (libjars.isEmpty() == false) {
        configuration.set(KEY_CONF_LIBRARIES, libjars);
    }
    return localUrls;
}

---

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private void configureJobJar(List<Path> paths, String className, Map<Path, Path> cacheMap) throws IOException {
    if (configuration.get(KEY_CONF_JAR) != null) {
        return;/*from  ww  w .  j av  a 2  s. c  o m*/
    }
    for (Path path : paths) {
        Path remote = cacheMap.get(path);
        URI uri = path.toUri();
        if (remote != null && uri.getScheme().equals("file")) { //$NON-NLS-1$
            File file = new File(uri);
            if (isInclude(file, className)) {
                Path qualified = remote.getFileSystem(configuration).makeQualified(remote);
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("Application class is in: file={2} ({1}), class={0}", //$NON-NLS-1$
                            className, path, qualified));
                }
                URI target = qualified.toUri();
                if (target.getScheme() != null
                        && (target.getScheme().equals("file") || target.getAuthority() != null)) { //$NON-NLS-1$
                    configuration.set(KEY_CONF_JAR, qualified.toString());
                }
                break;
            }
        }
    }
}

---

From source file:com.asakusafw.runtime.stage.optimizer.LibraryCopySuppressionConfigurator.java

License:Apache License

static String selectLibraries(String libraries) {
    String minLibrary = null;//from  w ww .j  a  v a2  s  .c o m
    long minSize = Long.MAX_VALUE;
    for (String library : libraries.split(",")) { //$NON-NLS-1$
        Path path = new Path(library);
        String scheme = path.toUri().getScheme();
        if (scheme != null && scheme.equals("file")) { //$NON-NLS-1$
            File file = new File(path.toUri());
            long size = file.length();
            if (size < minSize) {
                minLibrary = library;
                minSize = size;
            }
        }
    }
    if (minLibrary != null) {
        return minLibrary;
    }
    return libraries;
}

---

From source file:com.asakusafw.runtime.util.cache.FileCacheRepositoryTestRoot.java

License:Apache License

/**
 * Converts {@link Path} into {@link File}.
 * @param path the original path//  w w  w  .j a  va2 s  .com
 * @return the converted file
 */
protected File file(Path path) {
    return new File(path.toUri());
}

---

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

/**
 * Creates a new instance.// ww  w  . j av  a 2s  .  c  o m
 * @param configuration the current configuration
 * @param repository the cache root path (must be absolute)
 * @param lockProvider the cache lock provider
 * @param retryStrategy the retry strategy
 */
public HadoopFileCacheRepository(Configuration configuration, Path repository,
        LockProvider<? super Path> lockProvider, RetryStrategy retryStrategy) {
    if (repository.toUri().getScheme() == null) {
        throw new IllegalArgumentException(
                MessageFormat.format("Cache repository location must contan the scheme: {0}", repository));
    }
    this.configuration = configuration;
    this.repository = repository;
    this.lockProvider = lockProvider;
    this.retryStrategy = retryStrategy;
    this.checkBeforeDelete = configuration.getBoolean(KEY_CHECK_BEFORE_DELETE, DEFAULT_CHECK_BEFORE_DELETE);
}

---

From source file:com.asakusafw.runtime.util.cache.MockFileCacheRepository.java

License:Apache License

@Override
public Path resolve(Path file) throws IOException, InterruptedException {
    URI uri = file.toUri();
    if (uri.getScheme() == null || uri.getScheme().equals("file") == false) {
        return null;
    }/*from  w ww. j  ava  2 s. c o  m*/
    File source = new File(uri);
    File target = new File(repository, source.getName());
    try (InputStream in = new FileInputStream(source); OutputStream out = new FileOutputStream(target);) {
        byte[] buf = new byte[256];
        while (true) {
            int read = in.read(buf);
            if (read < 0) {
                break;
            }
            out.write(buf, 0, read);
        }
    }
    return new Path(target.toURI());
}

---

From source file:com.asakusafw.thundergate.runtime.cache.mapreduce.CacheBuildClient.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 4) {
        throw new IllegalArgumentException(
                MessageFormat.format("Invalid arguments: {0}", Arrays.toString(args)));
    }// w  w w  .  j  av a  2 s  .c  om
    String subcommand = args[0];
    boolean create;
    if (subcommand.equals(SUBCOMMAND_CREATE)) {
        create = true;
    } else if (subcommand.equals(SUBCOMMAND_UPDATE)) {
        create = false;
    } else {
        throw new IllegalArgumentException(
                MessageFormat.format("Invalid arguments (unknown subcommand): {0}", Arrays.toString(args)));
    }

    Path cacheDirectory = new Path(args[1]);
    modelClass = getConf().getClassByName(args[2]);
    tableName = args[3];
    this.storage = new CacheStorage(getConf(), cacheDirectory.toUri());
    try {
        clearNext();
        if (create) {
            create();
        } else if (PatchStrategy.isTableJoin(tableName, storage)) {
            updateTable();
        } else {
            updateMerge();
        }
        switchHead();
    } finally {
        storage.close();
    }
    return 0;
}

---

From source file:com.bah.lucene.hdfs.SoftlinkHdfsDirectory.java

License:Apache License

private void createLinkForNewFile(String name) throws IOException {
    String uuid = UUID.randomUUID().toString();
    Path dataPath = new Path(_storePath, uuid);
    createLinkForNewFile(name, dataPath.toUri());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »