Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path) 

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:co.cask.tigon.data.hbase.HBaseTestBase.java

License:Apache License

public Path createHBaseRootDir(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path hbaseRootdir = new Path(fs.makeQualified(fs.getHomeDirectory()), "hbase");
    conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
    fs.mkdirs(hbaseRootdir);/*from   w w  w .j a v  a 2  s  .  com*/
    FSUtils.setVersion(fs, hbaseRootdir);
    return hbaseRootdir;
}

---

From source file:com.anhth12.lambda.BatchUpdateFunction.java

private static String joinFSPaths(FileSystem fs, FileStatus[] statuses) {
    StringBuilder sb = new StringBuilder();
    for (FileStatus status : statuses) {
        if (sb.length() > 0) {
            sb.append(",");
        }/*from w ww.  ja v a2s . c  om*/
        Path path = fs.makeQualified(status.getPath());
        sb.append(StringUtils.escapeString(path.toString()));
    }
    return sb.toString();
}

---

From source file:com.asakusafw.bulkloader.common.FileNameUtil.java

License:Apache License

/**
 * Resolves the raw path.// www .j  a v a  2  s  .com
 * @param conf current configuration
 * @param rawPaths raw paths
 * @param executionId current execution ID
 * @param user current user name
 * @return the resolved full path
 * @throws BulkLoaderSystemException if failed to resolve the path
 * @since 0.4.0
 */
public static List<Path> createPaths(Configuration conf, List<String> rawPaths, String executionId, String user)
        throws BulkLoaderSystemException {
    String basePathString = ConfigurationLoader.getProperty(Constants.PROP_KEY_BASE_PATH);
    Path basePath;
    if (basePathString == null || basePathString.isEmpty()) {
        basePath = null;
    } else {
        basePath = new Path(basePathString);
    }
    VariableTable variables = Constants.createVariableTable();
    variables.defineVariable(Constants.HDFS_PATH_VARIABLE_USER, user);
    variables.defineVariable(Constants.HDFS_PATH_VARIABLE_EXECUTION_ID, executionId);
    FileSystem fs;
    try {
        if (basePath == null) {
            fs = FileSystem.get(conf);
        } else {
            fs = FileSystem.get(basePath.toUri(), conf);
            basePath = fs.makeQualified(basePath);
        }
    } catch (IOException e) {
        throw new BulkLoaderSystemException(e, CLASS, "TG-COMMON-00019", rawPaths);
    }
    List<Path> results = new ArrayList<>();
    for (String rawPath : rawPaths) {
        String resolved = variables.parse(rawPath, false);
        Path fullPath;
        if (basePath == null) {
            fullPath = fs.makeQualified(new Path(resolved));
        } else {
            fullPath = new Path(basePath, resolved);
        }
        results.add(fullPath);
    }
    return results;
}

---

From source file:com.asakusafw.compiler.fileio.HadoopFileIoProcessorTest.java

License:Apache License

private <T> ModelOutput<T> openOutput(Class<T> aClass, Location location) throws IOException {
    Configuration conf = tester.configuration();
    Path path = new Path(location.toPath('/'));
    FileSystem fs = path.getFileSystem(conf);
    SequenceFile.Writer writer = SequenceFile.createWriter(conf,
            SequenceFile.Writer.file(fs.makeQualified(path)), SequenceFile.Writer.keyClass(NullWritable.class),
            SequenceFile.Writer.valueClass(aClass));
    return new SequenceFileModelOutput<>(writer);
}

---

From source file:com.asakusafw.compiler.fileio.HadoopFileIoProcessorTest.java

License:Apache License

private List<Ex1> getList(Class<? extends FileExporterDescription> exporter) {
    try {//ww w.j a  va2 s.c o m
        FileExporterDescription instance = exporter.newInstance();
        Path path = new Path(Location.fromPath(instance.getPathPrefix(), '/').toString());
        FileSystem fs = path.getFileSystem(tester.configuration());
        FileStatus[] statuses = fs.globStatus(path);
        List<Ex1> results = new ArrayList<>();
        for (FileStatus status : statuses) {
            try (SequenceFile.Reader reader = new SequenceFile.Reader(tester.configuration(),
                    SequenceFile.Reader.file(fs.makeQualified(status.getPath())))) {
                Ex1 model = new Ex1();
                while (reader.next(NullWritable.get(), model)) {
                    Ex1 copy = new Ex1();
                    copy.copyFrom(model);
                    results.add(copy);
                }
            }
        }
        Collections.sort(results, new Comparator<Ex1>() {
            @Override
            public int compare(Ex1 o1, Ex1 o2) {
                return o1.getSidOption().compareTo(o2.getSidOption());
            }
        });
        return results;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

---

From source file:com.asakusafw.lang.compiler.extension.testdriver.InternalExporterRetriever.java

License:Apache License

@Override
public void truncate(InternalExporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting output directory: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    Path output = path.getParent();
    Path target;/*ww  w.  j  a  v  a 2 s  . c  o m*/
    if (output == null) {
        LOG.warn(MessageFormat.format("skipped deleting output directory because it is a base directory: {0}",
                path));
        target = fs.makeQualified(path);
    } else {
        LOG.debug("output directory will be deleted: {}", output); //$NON-NLS-1$
        target = fs.makeQualified(output);
    }
    LOG.debug("deleting output target: {}", target); //$NON-NLS-1$
    try {
        FileStatus[] stats = fs.globStatus(path);
        for (FileStatus s : stats) {
            Path f = s.getPath();
            boolean deleted = fs.delete(f, true);
            LOG.debug("deleted output target (succeed={}): {}", deleted, f); //$NON-NLS-1$
        }
    } catch (IOException e) {
        LOG.debug("exception in truncate", e);
    }
}

---

From source file:com.asakusafw.lang.compiler.extension.testdriver.InternalImporterPreparator.java

License:Apache License

@Override
public void truncate(InternalImporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting input: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path target = fs.makeQualified(new Path(resolved));
    FileStatus[] stats = fs.globStatus(target);
    if (stats == null || stats.length == 0) {
        return;//from ww w  .  jav a2 s  .  com
    }
    for (FileStatus s : stats) {
        Path path = s.getPath();
        LOG.debug("deleting file: {}", path); //$NON-NLS-1$
        boolean succeed = fs.delete(path, true);
        LOG.debug("deleted file (succeed={}): {}", succeed, path); //$NON-NLS-1$
    }
    return;
}

---

From source file:com.asakusafw.operation.tools.directio.file.AbstractFileCopyCommand.java

License:Apache License

private Path qualify(Path path) {
    FileSystem fs = dataSourceParameter.getHadoopFileSystem(path);
    return fs.makeQualified(path);
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceProfileTest.java

License:Apache License

/**
 * convert with relative path.//from www  .java 2 s .  co m
 * @throws Exception if failed
 */
@Test
public void convert_relpath() throws Exception {
    Map<String, String> attributes = new HashMap<>();
    attributes.put(KEY_PATH, "relative");
    DirectDataSourceProfile profile = new DirectDataSourceProfile("testing", HadoopDataSource.class, "context",
            attributes);
    Configuration conf = new Configuration();
    HadoopDataSourceProfile result = HadoopDataSourceProfile.convert(profile, conf);

    FileSystem defaultFs = FileSystem.get(conf);
    Path path = defaultFs.makeQualified(new Path(defaultFs.getWorkingDirectory(), "relative"));
    assertThat(result.getFileSystem().getCanonicalServiceName(), is(defaultFs.getCanonicalServiceName()));
    assertThat(result.getFileSystemPath(), is(path));
}

---

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static void move(Counter counter, FileSystem fromFs, Path from, FileSystem toFs, Path to,
        boolean fromLocal) throws IOException {
    if (counter == null) {
        throw new IllegalArgumentException("counter must not be null"); //$NON-NLS-1$
    }//from w ww.  j a va  2s  .c om
    if (fromFs == null) {
        throw new IllegalArgumentException("fromFs must not be null"); //$NON-NLS-1$
    }
    if (from == null) {
        throw new IllegalArgumentException("from must not be null"); //$NON-NLS-1$
    }
    if (toFs == null) {
        throw new IllegalArgumentException("toFs must not be null"); //$NON-NLS-1$
    }
    if (to == null) {
        throw new IllegalArgumentException("to must not be null"); //$NON-NLS-1$
    }
    if (fromLocal && isLocalPath(from) == false) {
        throw new IllegalArgumentException("from must be on local file system"); //$NON-NLS-1$
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start moving files (from={0}, to={1})", //$NON-NLS-1$
                from, to));
    }
    Path source = fromFs.makeQualified(from);
    Path target = toFs.makeQualified(to);
    List<Path> list = createFileListRelative(counter, fromFs, source);
    if (list.isEmpty()) {
        return;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Process moving files (from={0}, to={1}, count={2})", //$NON-NLS-1$
                from, to, list.size()));
    }
    Set<Path> directoryCreated = new HashSet<>();
    for (Path path : list) {
        Path sourceFile = new Path(source, path);
        Path targetFile = new Path(target, path);
        if (LOG.isTraceEnabled()) {
            FileStatus stat = fromFs.getFileStatus(sourceFile);
            LOG.trace(MessageFormat.format("Moving file (from={0}, to={1}, size={2})", //$NON-NLS-1$
                    sourceFile, targetFile, stat.getLen()));
        }
        try {
            FileStatus stat = toFs.getFileStatus(targetFile);
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Deleting file: {0}", //$NON-NLS-1$
                        targetFile));
            }
            if (FileSystemCompatibility.isDirectory(stat)) {
                toFs.delete(targetFile, true);
            } else {
                toFs.delete(targetFile, false);
            }
        } catch (FileNotFoundException e) {
            Path targetParent = targetFile.getParent();
            if (directoryCreated.contains(targetParent) == false) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("Creating directory: {0}", //$NON-NLS-1$
                            targetParent));
                }
                toFs.mkdirs(targetParent);
                directoryCreated.add(targetParent);
            }
        }
        counter.add(1);
        if (fromLocal) {
            toFs.moveFromLocalFile(sourceFile, targetFile);
        } else {
            boolean succeed = toFs.rename(sourceFile, targetFile);
            if (succeed == false) {
                throw new IOException(
                        MessageFormat.format("Failed to move file (from={0}, to={1})", sourceFile, targetFile));
            }
        }
        counter.add(1);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish moving files (from={0}, to={1}, count={2})", //$NON-NLS-1$
                from, to, list.size()));
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »