Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.facebook.hive.orc.TestInputOutputFormat.java

License:Apache License

@Before
public void openFileSystem() throws Exception {
    conf = new JobConf();
    fs = FileSystem.getLocal(conf);
    testFilePath = new Path(workDir, "TestInputOutputFormat." + testCaseName.getMethodName() + ".orc");
    fs.delete(testFilePath, false);//from  www .  ja  v  a 2 s  .co  m
}

From source file:com.facebook.hive.orc.TestMapTreeWriter.java

License:Apache License

@Before
public void openFileSystem() throws Exception {
    conf = new Configuration();
    fs = FileSystem.getLocal(conf);
    testFilePath = new Path(workDir, "TestOrcFile." + testCaseName.getMethodName() + ".orc");
    testFilePath2 = new Path(workDir, "TestOrcFile2." + testCaseName.getMethodName() + ".orc");
    fs.delete(testFilePath, false);//from  w w  w  .j av a  2s  . c o  m
    fs.delete(testFilePath2, false);
}

From source file:com.fullcontact.cassandra.io.compress.CompressedRandomAccessReaderTest.java

License:Apache License

@BeforeClass
public static void setUp() throws IOException {
    fs = FileSystem.getLocal(new Configuration());
}

From source file:com.fullcontact.sstable.index.SSTableIndexIndexerTest.java

License:Apache License

@Before
public void setUp() throws Exception {
    fileSystem = FileSystem.getLocal(new Configuration());
}

From source file:com.google.cloud.dataflow.contrib.sorter.ExternalSorter.java

License:Apache License

/**
 * Initializes the hadoop sorter. Does some local file system setup, and is somewhat expensive
 * (~20 ms on local machine). Only executed when necessary.
 *//*from w  w w.  j  a v a 2  s. c o  m*/
private void initHadoopSorter() throws IOException {
    if (!initialized) {
        tempDir = new Path(options.getTempLocation(), "tmp" + UUID.randomUUID().toString());
        paths = new Path[] { new Path(tempDir, "test.seq") };

        JobConf conf = new JobConf();
        writer = SequenceFile.createWriter(conf, Writer.valueClass(BytesWritable.class),
                Writer.keyClass(BytesWritable.class), Writer.file(paths[0]),
                Writer.compression(CompressionType.NONE));

        FileSystem fs = FileSystem.getLocal(conf);
        sorter = new SequenceFile.Sorter(fs, new BytesWritable.Comparator(), BytesWritable.class,
                BytesWritable.class, conf);
        sorter.setMemory(options.getMemoryMB() * 1024 * 1024);

        initialized = true;
    }
}

From source file:com.google.mr4c.sources.MapFileSourceLocalTest.java

License:Open Source License

@Before
public void setUp() throws Exception {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    m_src = new MapFileSource(fs, new Path(fs.getWorkingDirectory(), m_dir));
    m_tester = new ArchiveSourceTester();
}

From source file:com.hadoop.compression.lzo.TestLzopOutputStream.java

License:Open Source License

@Override
protected void setUp() throws Exception {
    super.setUp();
    inputDataPath = System.getProperty("test.build.data", "data");
    Configuration conf = new Configuration();
    conf.set("io.compression.codecs", LzopCodec.class.getName());
    localFs = FileSystem.getLocal(conf).getRaw();
}

From source file:com.hadoop.mapreduce.TestLzoTextInputFormat.java

License:Open Source License

/**
 * Generate random data, compress it, index and md5 hash the data.
 * Then read it all back and md5 that too, to verify that it all went ok.
 * /*from  w  ww .  j  a  va 2s. c o  m*/
 * @param testWithIndex Should we index or not?
 * @param charsToOutput How many characters of random data should we output.
 * @throws IOException
 * @throws NoSuchAlgorithmException
 * @throws InterruptedException
 */
private void runTest(boolean testWithIndex, int charsToOutput)
        throws IOException, NoSuchAlgorithmException, InterruptedException {

    if (!GPLNativeCodeLoader.isNativeCodeLoaded()) {
        LOG.warn("Cannot run this test without the native lzo libraries");
        return;
    }

    Configuration conf = new Configuration();
    conf.setLong("fs.local.block.size", charsToOutput / 2);
    // reducing block size to force a split of the tiny file
    conf.set("io.compression.codecs", LzopCodec.class.getName());

    FileSystem localFs = FileSystem.getLocal(conf);
    localFs.delete(outputDir, true);
    localFs.mkdirs(outputDir);

    Job job = new Job(conf);
    TextOutputFormat.setCompressOutput(job, true);
    TextOutputFormat.setOutputCompressorClass(job, LzopCodec.class);
    TextOutputFormat.setOutputPath(job, outputDir);

    TaskAttemptContext attemptContext = new TaskAttemptContextImpl(job.getConfiguration(),
            new TaskAttemptID("123", 0, TaskType.REDUCE, 1, 2));

    // create some input data
    byte[] expectedMd5 = createTestInput(outputDir, localFs, attemptContext, charsToOutput);

    if (testWithIndex) {
        Path lzoFile = new Path(outputDir, lzoFileName);
        LzoTextInputFormat.createIndex(localFs, lzoFile);
    }

    LzoTextInputFormat inputFormat = new LzoTextInputFormat();
    TextInputFormat.setInputPaths(job, outputDir);

    List<InputSplit> is = inputFormat.getSplits(job);
    //verify we have the right number of lzo chunks
    if (testWithIndex && OUTPUT_BIG == charsToOutput) {
        assertEquals(3, is.size());
    } else {
        assertEquals(1, is.size());
    }

    // let's read it all and calculate the md5 hash
    for (InputSplit inputSplit : is) {
        RecordReader<LongWritable, Text> rr = inputFormat.createRecordReader(inputSplit, attemptContext);
        rr.initialize(inputSplit, attemptContext);

        while (rr.nextKeyValue()) {
            Text value = rr.getCurrentValue();

            md5.update(value.getBytes(), 0, value.getLength());
        }

        rr.close();
    }

    localFs.close();
    assertTrue(Arrays.equals(expectedMd5, md5.digest()));
}

From source file:com.hazelcast.jet.connector.hadoop.ReadHdfsPTest.java

License:Open Source License

private static Path writeToFile(String... values) throws IOException {
    LocalFileSystem local = FileSystem.getLocal(new Configuration());
    Path path = new Path(randomString());
    local.createNewFile(path);/*www .j ava2 s  . co m*/
    FSDataOutputStream outputStream = local.create(path);
    local.deleteOnExit(path);
    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream));
    for (String value : values) {
        writer.write(value);
    }
    writer.flush();
    writer.close();
    outputStream.close();
    return path;
}

From source file:com.hazelcast.jet.connector.hadoop.WriteHdfsPTest.java

License:Open Source License

public Path getPath() throws IOException {
    LocalFileSystem local = FileSystem.getLocal(new Configuration());
    Path path = new Path(randomString());
    local.mkdirs(path);// ww  w .j a  va 2s. c  o  m
    local.deleteOnExit(path);
    return path;
}