Example usage for org.apache.hadoop.fs Path getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.asakusafw.dag.compiler.extension.internalio.testing.InternalIoTestHelper.java

License:Apache License

/**
 * Collects output data.//from  w  w  w .  j a v  a  2 s  .c om
 * @param <T> the data type
 * @param dataType the data type
 * @param path the input path
 * @param action the prepare action
 */
public <T extends Writable> void output(Class<T> dataType, String path, Action<List<T>, ?> action) {
    Configuration conf = new Configuration();
    Path p = new Path(locate(path).toURI());
    try {
        FileSystem fs = p.getFileSystem(conf);
        FileStatus[] stats = fs.globStatus(p);
        List<T> results = new ArrayList<>();
        for (FileStatus stat : stats) {
            try (ModelInput<T> in = TemporaryStorage.openInput(conf, dataType, stat.getPath())) {
                while (true) {
                    T buf = dataType.newInstance();
                    if (in.readTo(buf) == false) {
                        break;
                    }
                    results.add(buf);
                }
            }
        }
        action.perform(results);
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

From source file:com.asakusafw.dag.runtime.directio.TransactionManager.java

License:Apache License

private void setTransactionInfo(boolean value) throws IOException {
    Path transactionInfo = getTransactionInfoPath();
    FileSystem fs = transactionInfo.getFileSystem(configuration);
    if (value) {/*from   ww  w .  j a v  a 2  s.c  o m*/
        try (OutputStream output = new SafeOutputStream(fs.create(transactionInfo, false));
                PrintWriter writer = new PrintWriter(
                        new OutputStreamWriter(output, HadoopDataSourceUtil.COMMENT_CHARSET))) {
            for (Map.Entry<String, String> entry : transactionProperties.entrySet()) {
                if (entry.getValue() != null) {
                    writer.printf("%s: %s%n", //$NON-NLS-1$
                            entry.getKey(), entry.getValue());
                }
            }
        }
    } else {
        fs.delete(transactionInfo, false);
    }
}

From source file:com.asakusafw.dag.runtime.directio.TransactionManager.java

License:Apache License

private void setCommitted(boolean value) throws IOException {
    Path commitMark = getCommitMarkPath();
    FileSystem fs = commitMark.getFileSystem(configuration);
    if (value) {//from w w  w  .  java2 s .c o m
        fs.create(commitMark, false).close();
    } else {
        fs.delete(commitMark, false);
    }
}

From source file:com.asakusafw.dag.runtime.directio.TransactionManager.java

License:Apache License

boolean isCommitted() throws IOException {
    Path commitMark = getCommitMarkPath();
    FileSystem fs = commitMark.getFileSystem(configuration);
    return fs.exists(commitMark);
}

From source file:com.asakusafw.dag.runtime.internalio.HadoopInternalInputTaskInfoTest.java

License:Apache License

/**
 * simple case.//from   w w w. j  a va 2  s  .  c om
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    Configuration conf = new Configuration();
    Path file = new Path(temporary.newFile().toURI());
    FileSystem fileSystem = file.getFileSystem(conf);
    put(fileSystem, file, "Hello, world!");

    List<String> results = new ArrayList<>();
    HadoopInternalInputTaskInfo<Text> info = new HadoopInternalInputTaskInfo<>(fileSystem, file, 0, 0,
            Text::new);
    try (ModelInput<Text> in = info.open()) {
        Text buf = info.newDataObject();
        while (in.readTo(buf)) {
            results.add(buf.toString());
        }
    }
    assertThat(results, containsInAnyOrder("Hello, world!"));
}

From source file:com.asakusafw.directio.tools.DirectIoCommandTestRoot.java

License:Apache License

/**
 * Creates a new indoubt transaction./* ww  w.  j a va 2  s .  c  o m*/
 * @param executionId target execution id
 * @throws IOException if failed
 * @throws InterruptedException if interrupted
 */
protected void indoubt(String executionId) throws IOException, InterruptedException {
    Path txPath = HadoopDataSourceUtil.getTransactionInfoPath(conf, executionId);
    Path cmPath = HadoopDataSourceUtil.getCommitMarkPath(conf, executionId);
    FileSystem fs = txPath.getFileSystem(conf);
    fs.create(txPath).close();
    fs.create(cmPath).close();
    int index = 0;
    for (String path : repo.getContainerPaths()) {
        String id = repo.getRelatedId(path);
        DirectDataSource ds = repo.getRelatedDataSource(path);
        OutputTransactionContext txContext = HadoopDataSourceUtil.createContext(executionId, id);
        OutputAttemptContext aContext = new OutputAttemptContext(txContext.getTransactionId(),
                String.valueOf(index), txContext.getOutputId(), new Counter());

        ds.setupTransactionOutput(txContext);
        ds.setupAttemptOutput(aContext);
        try (ModelOutput<StringBuilder> output = ds.openOutput(aContext,
                SimpleDataDefinition.newInstance(StringBuilder.class, new MockFormat()), "", executionId,
                new Counter())) {
            output.write(new StringBuilder("Hello, world!"));
        }
        ds.commitAttemptOutput(aContext);
        ds.cleanupAttemptOutput(aContext);

        index++;
    }
}

From source file:com.asakusafw.lang.compiler.extension.testdriver.TemporaryDataModelSource.java

License:Apache License

/**
 * Creates a new instance./*from w w  w. j  ava  2s .  c o m*/
 * @param conf current configuration
 * @param definition data type
 * @param pathExpression the source path (can include wildcard)
 * @throws IOException if failed to create instance
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
@SuppressWarnings("unchecked")
public TemporaryDataModelSource(Configuration conf, DataModelDefinition<?> definition, String pathExpression)
        throws IOException {
    this.conf = conf;
    this.definition = (DataModelDefinition<Object>) definition;
    this.object = definition.toObject(definition.newReflection().build());
    Path path = new Path(pathExpression);
    this.fs = path.getFileSystem(conf);
    FileStatus[] list = fs.globStatus(path);
    List<Path> paths = new ArrayList<>();
    for (int i = 0; i < list.length; i++) {
        paths.add(list[i].getPath());
    }
    this.rest = paths.iterator();
}

From source file:com.asakusafw.lang.compiler.mapreduce.testing.mock.WritableOutputFormat.java

License:Apache License

@Override
public RecordWriter<NullWritable, T> getRecordWriter(TaskAttemptContext context)
        throws IOException, InterruptedException {
    Path path = getDefaultWorkFile(context, null);
    FileSystem fs = path.getFileSystem(context.getConfiguration());
    return new Writer<>(new WritableModelOutput<T>(fs.create(path, true)));
}

From source file:com.asakusafw.m3bp.compiler.tester.externalio.TestIoTaskExecutor.java

License:Apache License

private <T extends Writable> void executeInput(String name, Class<T> dataType, List<Path> paths)
        throws IOException {
    Action<Object, Exception> action = inputs.get(name);
    Invariants.requireNonNull(action, () -> MessageFormat.format("missing input: {0}", name));
    Path path = new Path(paths.get(0).toString().replace('*', '_'));
    FileSystem fs = path.getFileSystem(configuration);
    try (ModelOutput<T> output = new TemporaryFileOutput<>(fs.create(path), dataType.getName(),
            OUTPUT_INIT_BUFFER_SIZE, OUTPUT_PAGE_SIZE)) {
        action.perform(output);//  w w w.j  a v  a2s  .c o m
    } catch (Error | RuntimeException | IOException e) {
        throw e;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}

From source file:com.asakusafw.m3bp.compiler.tester.externalio.TestIoTaskExecutor.java

License:Apache License

private <T extends Writable> void executeOutput(String name, Class<T> dataType, List<Path> paths)
        throws IOException {
    Action<Object, Exception> action = outputs.get(name);
    Invariants.requireNonNull(action, () -> MessageFormat.format("missing output: {0}", name));
    List<T> results = new ArrayList<>();
    for (Path pattern : paths) {
        FileSystem fs = pattern.getFileSystem(configuration);
        FileStatus[] stats = fs.globStatus(pattern);
        if (stats == null) {
            continue;
        }/*www. j av  a  2 s. co m*/
        for (FileStatus stat : stats) {
            try (ModelInput<T> in = new TemporaryFileInput<>(fs.open(stat.getPath()), 0)) {
                while (true) {
                    T instance = dataType.newInstance();
                    if (in.readTo(instance)) {
                        results.add(instance);
                    } else {
                        break;
                    }
                }
            } catch (Error | RuntimeException | IOException e) {
                throw e;
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        }
    }
    try {
        action.perform(results);
    } catch (Error | RuntimeException | IOException e) {
        throw e;
    } catch (Exception e) {
        throw new AssertionError(e);
    }
}