Example usage for org.apache.hadoop.fs FileStatus getPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getPath.

Prototype

public Path getPath() 

Source Link

Usage

From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormatTest.java

License:Apache License

/**
 * Simple case for record readers.//from w  w  w  .  ja  va2s.c  om
 * @throws Exception if failed
 */
@Test
public void reader_simple() throws Exception {
    Configuration conf = new ConfigurationProvider().newInstance();
    FileStatus stat = write(conf, 1);
    try (RecordReader<NullWritable, Text> reader = TemporaryInputFormat.createRecordReader()) {
        reader.initialize(new FileSplit(stat.getPath(), 0, stat.getLen(), null),
                JobCompatibility.newTaskAttemptContext(conf, id()));

        assertThat(reader.nextKeyValue(), is(true));
        assertThat(reader.getCurrentValue(), is(new Text("Hello, world!")));

        assertThat(reader.nextKeyValue(), is(false));
        assertThat((double) reader.getProgress(), closeTo(1.0, 0.01));
    }
}

---

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Adds a resource path into the target job object.
 * @param job the target job// w  w  w.ja  v a2  s.  c  om
 * @param resourcePath the resource path expression (this must be accessible from task execution nodes)
 * @param resourceName the resource name
 * @throws IOException if failed to detect resources on the path
 * @throws IllegalArgumentException if some parameters are {@code null}
 */
public static void add(Job job, String resourcePath, String resourceName) throws IOException {
    if (job == null) {
        throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$
    }
    if (resourcePath == null) {
        throw new IllegalArgumentException("resourcePath must not be null"); //$NON-NLS-1$
    }
    if (resourceName == null) {
        throw new IllegalArgumentException("resourceName must not be null"); //$NON-NLS-1$
    }
    Configuration conf = job.getConfiguration();
    List<FileStatus> list = TemporaryStorage.listStatus(conf, new Path(resourcePath));
    if (list.isEmpty()) {
        throw new IOException(MessageFormat.format("Resource not found: {0}", resourcePath));
    }
    List<String> localNames = restoreStrings(conf, getLocalCacheNameKey(resourceName));
    List<String> remotePaths = restoreStrings(conf, getRemotePathKey(resourceName));
    long size = conf.getLong(KEY_SIZE, 0L);
    int index = localNames.size();
    for (FileStatus status : list) {
        String name = String.format("%s-%04d", resourceName, index++); //$NON-NLS-1$
        StringBuilder buf = new StringBuilder();
        buf.append(status.getPath().toString());
        buf.append('#');
        buf.append(name);
        String cachePath = buf.toString();

        remotePaths.add(status.getPath().toString());
        localNames.add(name);
        try {
            URI uri = new URI(cachePath);
            DistributedCache.addCacheFile(uri, conf);
        } catch (URISyntaxException e) {
            throw new IllegalStateException(e);
        }
        size += status.getLen();
    }
    conf.setStrings(getLocalCacheNameKey(resourceName), localNames.toArray(new String[localNames.size()]));
    conf.setStrings(getRemotePathKey(resourceName), remotePaths.toArray(new String[remotePaths.size()]));
    conf.setLong(KEY_SIZE, size);
    if (JobCompatibility.isLocalMode(job)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("symlinks for distributed cache will not be created in standalone mode"); //$NON-NLS-1$
        }
    } else {
        DistributedCache.createSymlink(conf);
    }
}

---

From source file:com.asakusafw.runtime.stage.temporary.TemporaryStorage.java

License:Apache License

/**
 * Resolves the raw path pattern into the concrete path list.
 * @param conf current configuration//www  .  ja v  a 2 s  .  c  o  m
 * @param pathPattern path pattern which describes temporary storage
 * @return the resolved paths
 * @throws IOException if failed to resolve path pattern
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public static List<Path> list(Configuration conf, Path pathPattern) throws IOException {
    if (conf == null) {
        throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$
    }
    if (pathPattern == null) {
        throw new IllegalArgumentException("pathPattern must not be null"); //$NON-NLS-1$
    }
    List<FileStatus> statusList = listStatus(conf, pathPattern);
    if (statusList.isEmpty()) {
        return Collections.emptyList();
    }
    List<Path> results = new ArrayList<>();
    for (FileStatus status : statusList) {
        results.add(status.getPath());
    }
    return results;
}

---

From source file:com.asakusafw.testdriver.mapreduce.io.TemporaryInputPreparator.java

License:Apache License

static void delete(FileSystem fs, Path target) throws IOException {
    FileStatus[] stats = fs.globStatus(target);
    if (stats == null || stats.length == 0) {
        return;/*from   w  w  w.j  a  v  a2s  .c  o  m*/
    }
    for (FileStatus s : stats) {
        Path path = s.getPath();
        LOG.debug("deleting file: {}", path); //$NON-NLS-1$
        boolean succeed = fs.delete(path, true);
        LOG.debug("deleted file (succeed={}): {}", succeed, path); //$NON-NLS-1$
    }
}

---

From source file:com.asakusafw.testdriver.testing.dsl.SimpleBatchAction.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    MacSnappyJavaWorkaround.install();/*from  w w w.  j  a  va  2  s . c o m*/
    FileSystem fs = FileSystem.get(getConf());
    fs.mkdirs(new Path(SimpleExporter.DIRECTORY));
    Path inputDir = new Path(SimpleImporter.DIRECTORY);
    int index = 0;
    for (FileStatus input : fs.listStatus(inputDir)) {
        Path output = new Path(SimpleExporter.OUTPUT_PREFIX + index++);
        process(input.getPath(), output);
    }
    extra();
    return 0;
}

---

From source file:com.asakusafw.testdriver.testing.moderator.MockExporterRetriever.java

License:Apache License

@Override
public void truncate(MockExporterDescription description, TestContext context) throws IOException {
    LOG.debug("deleting output directory: {}", description); //$NON-NLS-1$
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    Path path = new Path(description.getGlob());
    try {/*from  www.jav  a 2 s. c o m*/
        FileStatus[] stats = fs.globStatus(path);
        for (FileStatus s : stats) {
            fs.delete(s.getPath(), false);
        }
    } catch (IOException e) {
        LOG.debug("exception in truncate", e);
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.AbstractSshHadoopFsMirrorTest.java

License:Apache License

private void put(FileList.Writer writer, String path, String... contents) throws IOException {
    Configuration conf = new Configuration();
    File temp = folder.newFile(path);
    FileSystem fs = FileSystem.getLocal(conf);
    try (ModelOutput<Text> output = TemporaryStorage.openOutput(conf, Text.class, new Path(temp.toURI()))) {
        for (String content : contents) {
            output.write(new Text(content));
        }/*from ww  w.  j a v  a2s  . c  o  m*/
    }
    FileStatus status = fs.getFileStatus(new Path(temp.toURI()));
    try (FSDataInputStream src = fs.open(status.getPath());
            OutputStream dst = writer.openNext(status.getPath())) {
        byte[] buf = new byte[256];
        while (true) {
            int read = src.read(buf);
            if (read < 0) {
                break;
            }
            dst.write(buf, 0, read);
        }
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopDelete.java

License:Apache License

private void doDelete(FileSystem fs, FileStatus status, FileList.Writer drain) throws IOException {
    assert fs != null;
    assert status != null;
    assert drain != null;
    WGLOG.info("I22004", fs.getUri(), status.getPath());
    try (OutputStream output = drain.openNext(status.getPath())) {
        String failReason = null;
        try {//www  .  j a v  a2  s  .c  om
            boolean deleted;
            if (RuntimeContext.get().isSimulation()) {
                deleted = true;
            } else {
                deleted = fs.delete(status.getPath(), true);
            }
            if (deleted == false) {
                if (fs.exists(status.getPath())) {
                    WGLOG.warn("W22001", fs.getUri(), status.getPath());
                    failReason = "Unknown";
                }
            }
        } catch (IOException e) {
            WGLOG.warn(e, "W22001", fs.getUri(), status.getPath());
            failReason = e.toString();
        }
        if (failReason != null) {
            output.write(failReason.getBytes(UTF8));
        }
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGet.java

License:Apache License

private InputStream getInput(FileSystem fs, FileStatus status) throws IOException {
    if (RuntimeContext.get().isSimulation()) {
        return new VoidInputStream();
    } else {/*  www  . ja  v  a 2  s .  c  o  m*/
        return fs.open(status.getPath(), BUFFER_SIZE);
    }
}

---

From source file:com.asakusafw.windgate.hadoopfs.ssh.WindGateHadoopGet.java

License:Apache License

private void transfer(FileSystem fs, FileStatus status, InputStream input, Writer drain) throws IOException {
    assert fs != null;
    assert status != null;
    assert input != null;
    assert drain != null;
    WGLOG.info("I20004", fs.getUri(), status.getPath());
    long transferred = 0;
    try {/* ww  w  .  j av a 2 s  . co  m*/
        if (RuntimeContext.get().isSimulation() == false) {
            try (OutputStream output = drain.openNext(status.getPath())) {
                byte[] buf = new byte[1024];
                while (true) {
                    int read = input.read(buf);
                    if (read < 0) {
                        break;
                    }
                    output.write(buf, 0, read);
                    transferred += read;
                }
            }
        }
    } finally {
        input.close();
    }
    WGLOG.info("I20005", fs.getUri(), status.getPath(), transferred);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »