Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.asakusafw.directio.hive.parquet.ParquetFileFormatTest.java

License:Apache License

private <T> File save(ParquetFileFormat<T> format, List<T> values) throws IOException, InterruptedException {
    File file = folder.newFile();
    Assume.assumeThat(file.delete() || file.exists() == false, is(true));
    LocalFileSystem fs = FileSystem.getLocal(format.getConf());
    try (ModelOutput<T> output = format.createOutput(format.getSupportedType(), fs, new Path(file.toURI()),
            new Counter())) {
        for (T value : values) {
            output.write(value);//from   w  w w .  j  ava 2  s.co  m
        }
    }
    assertThat(file.exists(), is(true));
    return file;
}

From source file:com.asakusafw.directio.hive.parquet.ParquetFileFormatTest.java

License:Apache License

private <T> List<T> load(ParquetFileFormat<T> format, File file) throws IOException, InterruptedException {
    LocalFileSystem fs = FileSystem.getLocal(format.getConf());
    try (ModelInput<T> input = format.createInput(format.getSupportedType(), fs, new Path(file.toURI()), 0,
            file.length(), new Counter())) {
        List<T> results = new ArrayList<>();
        while (true) {
            @SuppressWarnings("unchecked")
            T value = (T) format.getDataModelDescriptor().createDataModelObject();
            if (input.readTo(value) == false) {
                break;
            }//  w  ww .j a va 2s. c o m
            results.add(value);
        }
        return results;
    }
}

From source file:com.asakusafw.operation.tools.directio.conf.ConfigurationSystemCommand.java

License:Apache License

@Override
public void run() {
    LOG.debug("starting {}", getClass().getSimpleName());

    Configuration conf = configurationParameter.getConfiguration();
    try (PrintWriter writer = outputParameter.open()) {
        try {//from   w  w  w .j ava  2  s.com
            org.apache.hadoop.fs.Path systemDir = getSystemDir(conf);
            org.apache.hadoop.fs.Path localTemp = HadoopDataSourceUtil
                    .getLocalTemporaryDirectory(FileSystem.getLocal(conf));
            writer.printf("configuration: %s%n",
                    configurationParameter.getPath().map(Path::toString).orElse("N/A"));
            writer.printf("system directory: %s%n", systemDir);
            writer.printf("local temporary: %s%n",
                    Optional.ofNullable(localTemp).map(it -> it.toString()).orElse("N/A"));
        } catch (IOException e) {
            LOG.warn("error occurred while loading system configuration", e);
        }
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceProfile.java

License:Apache License

/**
 * Creates a new instance.//from   w ww  . ja  v a 2s .c  o m
 * @param conf the current configuration
 * @param id the ID of this datasource
 * @param contextPath the logical context path
 * @param fileSystemPath the mapping target path
 * @param temporaryPath the temporary root path
 * @throws IOException if failed to create profile
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public HadoopDataSourceProfile(Configuration conf, String id, String contextPath, Path fileSystemPath,
        Path temporaryPath) throws IOException {
    this.id = id;
    this.contextPath = contextPath;
    this.fileSystemPath = fileSystemPath;
    this.temporaryPath = temporaryPath;
    this.fileSystem = fileSystemPath.getFileSystem(conf);
    this.localFileSystem = FileSystem.getLocal(conf);
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

/**
 * Returns the local temporary directory.
 * @param localFileSystem the local file system
 * @return the output path (must be on local fs), or {@code null} if not defined
 * @throws IOException if failed to compute the path
 * @throws IllegalArgumentException if some parameters were {@code null}
 *//*from  w ww .  ja  v  a  2 s.  c  o m*/
public static Path getLocalTemporaryDirectory(LocalFileSystem localFileSystem) throws IOException {
    if (localFileSystem == null) {
        throw new IllegalArgumentException("localFileSystem must not be null"); //$NON-NLS-1$
    }
    Configuration conf = localFileSystem.getConf();
    if (conf == null) {
        return null;
    }
    String path = conf.get(KEY_LOCAL_TEMPDIR);
    if (path == null) {
        return null;
    }
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path result = fs.makeQualified(new Path(path));
    return result;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtilTest.java

License:Apache License

private FileSystem getTempFileSystem() throws IOException {
    Configuration conf = new Configuration();
    LocalFileSystem local = FileSystem.getLocal(conf);
    return local;
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input./*from   ww w. j  a va  2s.com*/
 * @throws Exception if failed
 */
@Test
public void input() throws Exception {
    final int count = 10000;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, 0,
            fs.getFileStatus(path).getLen(), new Counter())) {
        StringOption value = new StringOption();
        for (int i = 0; i < count; i++) {
            String answer = "Hello, world at " + i;
            assertThat(answer, in.readTo(value), is(true));
            assertThat(value.getAsString(), is(answer));
        }
        assertThat("eof", in.readTo(value), is(false));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input.//w  ww.ja va2  s  .c om
 * @throws Exception if failed
 */
@Test
public void input_fragment() throws Exception {
    final int count = 30000;
    Random rand = new Random();
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    for (int attempt = 0; attempt < 5; attempt++) {
        int index = 0;
        long offset = 0;
        while (offset < fileLen) {
            long length = SequenceFile.SYNC_INTERVAL * (rand.nextInt(10) + 2);
            length = Math.min(length, fileLen - offset);
            try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                    new Counter())) {
                while (in.readTo(value)) {
                    String answer = "Hello, world at " + index;
                    assertThat(value.getAsString(), is(answer));
                    index++;
                }
                assertThat("eof", in.readTo(value), is(false));
            }
            offset += length;
        }
        assertThat(index, is(count));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input.//from w w  w.j  av a  2  s .  c  o m
 * @throws Exception if failed
 */
@Test
public void input_largerecord() throws Exception {
    StringBuilder buf = new StringBuilder();
    for (int i = 0; i < 1000000; i++) {
        buf.append("Hello, world!");
    }
    Text record = new Text(buf.toString());

    final int count = 5;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set(record);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    int index = 0;
    long offset = 0;
    while (offset < fileLen) {
        long length = SequenceFile.SYNC_INTERVAL * 2;
        length = Math.min(length, fileLen - offset);
        try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                new Counter())) {
            while (in.readTo(value)) {
                assertThat(value.get(), is(record));
                index++;
            }
            assertThat("eof", in.readTo(value), is(false));
        }
        offset += length;
    }
    assertThat(index, is(count));
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input empty file./*from  w  ww  .java  2 s .c o m*/
 * @throws Exception if failed
 */
@Test
public void input_empty() throws Exception {
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    fs.create(path).close();
    try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, 0,
            fs.getFileStatus(path).getLen(), new Counter())) {
        assertThat("eof", in.readTo(new StringOption()), is(false));
    }
}