Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:cascading.tap.hadoop.Lfs.java

License:Open Source License

@Override
protected FileSystem getFileSystem(Configuration conf) {
    try {/*from  ww  w .  j  a v  a 2s.  com*/
        return FileSystem.getLocal(conf);
    } catch (IOException exception) {
        throw new TapException("unable to get handle to get local filesystem", exception);
    }
}

From source file:cascading.tap.Lfs.java

License:Open Source License

protected FileSystem getFileSystem(JobConf conf) throws IOException {
    return FileSystem.getLocal(conf);
}

From source file:cascading.util.Util.java

License:Open Source License

public static Thread getHDFSShutdownHook() {
    Exception caughtException = null;

    try {//from   w  w w  . ja v  a  2s .  c  om
        // we must init the FS so the finalizer is registered
        FileSystem.getLocal(new JobConf());

        Field field = FileSystem.class.getDeclaredField("clientFinalizer");
        field.setAccessible(true);

        Thread finalizer = (Thread) field.get(null);

        if (finalizer != null)
            Runtime.getRuntime().removeShutdownHook(finalizer);

        return finalizer;
    } catch (NoSuchFieldException exception) {
        caughtException = exception;
    } catch (IllegalAccessException exception) {
        caughtException = exception;
    } catch (IOException exception) {
        caughtException = exception;
    }

    LOG.info("unable to find and remove client hdfs shutdown hook, received exception: "
            + caughtException.getClass().getName());

    return null;
}

From source file:clone.ReadSequenceFile.java

License:Apache License

public static void main(String[] args) throws IOException {
    if (args.length < 1) {
        System.out.println("args: [path] [max-num-of-records-per-file]");
        System.exit(-1);/*ww w.j av  a 2s . c  om*/
    }

    String f = args[0];

    int max = Integer.MAX_VALUE;
    if (args.length >= 2) {
        max = Integer.parseInt(args[1]);
    }

    boolean useLocal = args.length >= 3 && args[2].equals("local") ? true : false;

    if (useLocal) {
        System.out.println("Reading from local filesystem");
    }

    FileSystem fs = useLocal ? FileSystem.getLocal(new Configuration()) : FileSystem.get(new Configuration());
    Path p = new Path(f);

    if (fs.getFileStatus(p).isDir()) {
        readSequenceFilesInDir(p, fs, max);
    } else {
        readSequenceFile(p, fs, max);
    }
}

From source file:colossal.pipe.ColPipe.java

License:Apache License

public ColPipe(Class<?> jarClass) {
    baseConf.setJarByClass(jarClass);/*from   w  ww  .j a v  a  2s .c o  m*/
    baseConf.set("mapred.job.reuse.jvm.num.tasks", "-1");
    try {
        FileSystem fs = FileSystem.get(new URI("/"), baseConf);
        FileSystem localfs = FileSystem.getLocal(baseConf);
        if (fs.equals(localfs)) {
            baseConf.setNumReduceTasks(2); // run only 2 reducers for local
        } else {
            baseConf.setNumReduceTasks(32); // default to 32 reducers - need to tune this
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

From source file:com.asakusafw.directio.hive.orc.OrcFileFormatTest.java

License:Apache License

/**
 * I/O with fragment.//from w w w  .  ja  v  a2  s  .c om
 * @throws Exception if failed
 */
@Test
public void io_fragment() throws Exception {
    File file = folder.newFile();
    Assume.assumeThat(file.delete() || file.exists() == false, is(true));

    OrcFileFormat<MockSimple> format = format(MockSimple.class);
    LocalFileSystem fs = FileSystem.getLocal(format.getConf());
    try (ModelOutput<MockSimple> output = format.createOutput(MockSimple.class, fs, new Path(file.toURI()),
            new Counter());) {
        output.write(new MockSimple(100, "Hello, world!"));
    }
    assertThat(file.exists(), is(true));

    FileStatus stat = fs.getFileStatus(new Path(file.toURI()));
    List<DirectInputFragment> fragments = format.computeInputFragments(new StripedDataFormat.InputContext(
            MockSimple.class, Arrays.asList(stat), fs, -1L, -1L, false, false));

    assertThat(fragments, hasSize(1));
    DirectInputFragment first = fragments.get(0);

    try (ModelInput<MockSimple> input = format.createInput(MockSimple.class, fs, new Path(first.getPath()),
            first.getOffset(), first.getSize(), new Counter())) {
        MockSimple buf = new MockSimple();
        assertThat(input.readTo(buf), is(true));
        assertThat(buf.number, is(new IntOption(100)));
        assertThat(buf.string, is(new StringOption("Hello, world!")));

        assertThat(input.readTo(buf), is(false));
    }
}

From source file:com.asakusafw.directio.hive.orc.OrcFileFormatTest.java

License:Apache License

private <T> File save(OrcFileFormat<T> format, List<T> values) throws IOException, InterruptedException {
    File file = folder.newFile();
    Assume.assumeThat(file.delete() || file.exists() == false, is(true));
    LocalFileSystem fs = FileSystem.getLocal(format.getConf());
    try (ModelOutput<T> output = format.createOutput(format.getSupportedType(), fs, new Path(file.toURI()),
            new Counter())) {
        for (T value : values) {
            output.write(value);//from  w w  w  .  jav a2  s . c  o m
        }
    }
    assertThat(file.exists(), is(true));
    return file;
}

From source file:com.asakusafw.directio.hive.orc.OrcFileFormatTest.java

License:Apache License

private <T> List<T> load(OrcFileFormat<T> format, File file) throws IOException, InterruptedException {
    LocalFileSystem fs = FileSystem.getLocal(format.getConf());
    try (ModelInput<T> input = format.createInput(format.getSupportedType(), fs, new Path(file.toURI()), 0,
            file.length(), new Counter())) {
        List<T> results = new ArrayList<>();
        while (true) {
            @SuppressWarnings("unchecked")
            T value = (T) format.getDataModelDescriptor().createDataModelObject();
            if (input.readTo(value) == false) {
                break;
            }// w ww.  j a  va  2s . co  m
            results.add(value);
        }
        return results;
    }
}

From source file:com.asakusafw.directio.hive.parquet.ParquetFileFormatTest.java

License:Apache License

/**
 * I/O with fragment./* www.  j av  a2 s. c om*/
 * @throws Exception if failed
 */
@Test
public void io_fragment() throws Exception {
    File file = folder.newFile();
    Assume.assumeThat(file.delete() || file.exists() == false, is(true));

    ParquetFileFormat<MockSimple> format = format(MockSimple.class);
    LocalFileSystem fs = FileSystem.getLocal(format.getConf());
    try (ModelOutput<MockSimple> output = format.createOutput(MockSimple.class, fs, new Path(file.toURI()),
            new Counter())) {
        output.write(new MockSimple(100, "Hello, world!"));
    }
    assertThat(file.exists(), is(true));

    FileStatus stat = fs.getFileStatus(new Path(file.toURI()));
    List<DirectInputFragment> fragments = format.computeInputFragments(new StripedDataFormat.InputContext(
            MockSimple.class, Arrays.asList(stat), fs, -1L, -1L, false, false));

    assertThat(fragments, hasSize(1));
    DirectInputFragment first = fragments.get(0);

    try (ModelInput<MockSimple> input = format.createInput(MockSimple.class, fs, new Path(first.getPath()),
            first.getOffset(), first.getSize(), new Counter())) {
        MockSimple buf = new MockSimple();
        assertThat(input.readTo(buf), is(true));
        assertThat(buf.number, is(new IntOption(100)));
        assertThat(buf.string, is(new StringOption("Hello, world!")));

        assertThat(input.readTo(buf), is(false));
    }
}

From source file:com.asakusafw.directio.hive.parquet.ParquetFileFormatTest.java

License:Apache License

private <T> ModelInput<T> load(ParquetFileFormat<T> format, String name)
        throws IOException, InterruptedException {
    File target = folder.newFile();
    try (InputStream in = getClass().getResourceAsStream(name)) {
        assertThat(in, is(notNullValue()));
        IOUtils.copyBytes(in, new FileOutputStream(target), 1024, true);
    }//w  w  w.  ja  v a 2s. com
    FileSystem fs = FileSystem.getLocal(format.getConf());
    return format.createInput(format.getSupportedType(), fs, new Path(target.toURI()), 0, -1, new Counter());
}