Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.inmobi.messaging.consumer.hadoop.TestConsumerPartitionStartTime.java

License:Apache License

@AfterTest
public void cleanup() throws Exception {
    FileSystem lfs = FileSystem.getLocal(conf);
    for (Path rootDir : rootDirs) {
        LOG.debug("Cleaning up the dir: " + rootDir.getParent());
        lfs.delete(rootDir.getParent(), true);
    }/*from  w  w  w . j a v  a2 s .  com*/
    lfs.delete(new Path(chkpointPath).getParent(), true);
}

From source file:com.inmobi.messaging.consumer.hadoop.TestHadoopConsumerWithPartitionList.java

License:Apache License

@AfterTest
public void cleanup() throws IOException {
    FileSystem lfs = FileSystem.getLocal(conf);
    for (Path rootDir : rootDirs) {
        LOG.debug("Cleaning up the dir: " + rootDir);
        lfs.delete(rootDir.getParent(), true);
    }/*www .j  a  va 2 s  . c  o m*/
    // delete checkpoint dir
    lfs.delete(new Path(ck1).getParent(), true);
}

From source file:com.inmobi.messaging.consumer.util.MiniClusterUtil.java

License:Apache License

public static synchronized MiniDFSCluster getDFSCluster(Configuration conf) throws IOException {
    if (dfsCluster == null) {
        lfs = FileSystem.getLocal(conf);
        lfs.delete(new Path(MiniClusterUtil.getBaseDirectory().toString()), true);
        dfsCluster = new MiniDFSCluster(conf, 1, true, null);
    }/*from www . j  av  a  2s .  co  m*/
    numAccess++;
    return dfsCluster;
}

From source file:com.knewton.mapreduce.SSTableColumnRecordReaderTest.java

License:Apache License

@Test
public void testNextKeyValue() throws Exception {
    Path inputPath = inputSplit.getPath();
    FileSystem remoteFS = FileSystem.get(inputPath.toUri(), conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    TaskAttemptContext context = getTaskAttemptContext();
    ssTableColumnRecordReader.initialize(inputSplit, context);
    verify(ssTableColumnRecordReader).copyTablesToLocal(remoteFS, localFS, inputPath, context);

    assertEquals(0, ssTableColumnRecordReader.getProgress(), 0);
    assertTrue(ssTableColumnRecordReader.nextKeyValue());
    assertEquals(key.getKey(), ssTableColumnRecordReader.getCurrentKey());
    assertEquals(value, ssTableColumnRecordReader.getCurrentValue());

    assertEquals(0.5, ssTableColumnRecordReader.getProgress(), 0);
    assertTrue(ssTableColumnRecordReader.nextKeyValue());
    assertEquals(key.getKey(), ssTableColumnRecordReader.getCurrentKey());
    assertEquals(value, ssTableColumnRecordReader.getCurrentValue());

    assertEquals(1, ssTableColumnRecordReader.getProgress(), 0);
    assertFalse(ssTableColumnRecordReader.nextKeyValue());
    assertNull(ssTableColumnRecordReader.getCurrentKey());
    assertNull(ssTableColumnRecordReader.getCurrentValue());
}

From source file:com.knewton.mapreduce.SSTableRecordReader.java

License:Apache License

/**
 * Performs all the necessary actions to initialize and prepare this record reader.
 *///  w  w  w . ja v  a 2s  . c  o m
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
        throws IOException, InterruptedException {
    this.ctx = context;
    conf = context.getConfiguration();
    keysRead = 0;
    components = Sets.newHashSetWithExpectedSize(3);
    FileSplit split = (FileSplit) inputSplit;
    validateConfiguration(conf);

    // Get comparator. Subcomparator can be null.
    AbstractType<?> comparator = getConfComparator(conf);
    AbstractType<?> subcomparator = getConfSubComparator(conf);

    // Get partitioner for keys
    IPartitioner partitioner = getConfPartitioner(conf);

    // Move minimum required db tables to local disk.
    Path dataTablePath = split.getPath();
    FileSystem remoteFS = FileSystem.get(dataTablePath.toUri(), conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    copyTablesToLocal(remoteFS, localFS, dataTablePath, context);
    CFMetaData cfMetaData;
    if (getConfIsSparse(conf)) {
        cfMetaData = CFMetaData.sparseCFMetaData(getDescriptor().ksname, getDescriptor().cfname, comparator);
    } else {
        cfMetaData = CFMetaData.denseCFMetaData(getDescriptor().ksname, getDescriptor().cfname, comparator,
                subcomparator);
    }
    // Open table and get scanner
    SSTableReader tableReader = openSSTableReader(partitioner, cfMetaData);
    setTableScanner(tableReader);
}

From source file:com.knewton.mapreduce.SSTableRecordReaderTest.java

License:Apache License

/**
 * Test a valid configuration of the SSTableColumnRecordReader.
 *//*from w  w  w  .ja  va  2s  . co  m*/
@Test
public void testInitializeColumnReader() throws Exception {
    Path inputPath = inputSplit.getPath();
    FileSystem remoteFS = FileSystem.get(inputPath.toUri(), conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    TaskAttemptContext context = getTaskAttemptContext(true, true, false);
    ssTableColumnRecordReader.initialize(inputSplit, context);
    verify(ssTableColumnRecordReader).copyTablesToLocal(remoteFS, localFS, inputPath, context);
    ssTableColumnRecordReader.close();
}

From source file:com.knewton.mapreduce.SSTableRecordReaderTest.java

License:Apache License

/**
 * Test a valid configuration of the SSTableRowRecordReader.
 *//*from  www  .  ja  va2  s .  co  m*/
@Test
public void testInitializeRowReader() throws Exception {
    Path inputPath = inputSplit.getPath();
    FileSystem remoteFS = FileSystem.get(inputPath.toUri(), conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    TaskAttemptContext context = getTaskAttemptContext(true, true, true);
    ssTableRowRecordReader.initialize(inputSplit, context);
    verify(ssTableRowRecordReader).copyTablesToLocal(remoteFS, localFS, inputPath, context);
    ssTableRowRecordReader.close();
}

From source file:com.knewton.mapreduce.SSTableRecordReaderTest.java

License:Apache License

/**
 * Tests to make sure initialization doesn't fail when a sparse CF is specified
 *//*from  ww w .  j av a 2 s  .c  o  m*/
@Test
public void testInitialize() throws Exception {
    Path inputPath = inputSplit.getPath();
    FileSystem remoteFS = FileSystem.get(inputPath.toUri(), conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    TaskAttemptContext context = getTaskAttemptContext(true, true, true);
    SSTableInputFormat.setIsSparse(false, job);
    ssTableColumnRecordReader.initialize(inputSplit, context);
    verify(ssTableColumnRecordReader).copyTablesToLocal(remoteFS, localFS, inputPath, context);
    ssTableColumnRecordReader.close();
}

From source file:com.knewton.mapreduce.SSTableRowRecordReaderTest.java

License:Apache License

/**
 * Make sure that the single row that was set can be read and returned
 *//*w  ww.j  av a 2  s  .  c  o  m*/
@Test
public void testNextKeyValue() throws Exception {
    Path inputPath = inputSplit.getPath();
    FileSystem remoteFS = FileSystem.get(inputPath.toUri(), conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    TaskAttemptContext context = getTaskAttemptContext();
    ssTableRowRecordReader.initialize(inputSplit, context);
    verify(ssTableRowRecordReader).copyTablesToLocal(remoteFS, localFS, inputPath, context);

    assertEquals(0, ssTableRowRecordReader.getProgress(), 0);
    assertTrue(ssTableRowRecordReader.nextKeyValue());
    assertEquals(key.getKey(), ssTableRowRecordReader.getCurrentKey());
    assertEquals(row, ssTableRowRecordReader.getCurrentValue());

    assertEquals(1, ssTableRowRecordReader.getProgress(), 0);
    assertFalse(ssTableRowRecordReader.nextKeyValue());
    assertNull(ssTableRowRecordReader.getCurrentKey());
    assertNull(ssTableRowRecordReader.getCurrentValue());
}

From source file:com.lightboxtechnologies.spectrum.ExtractDataMapper.java

License:Apache License

SequenceFile.Reader openExtentsFile(FileSystem hdpFs, Configuration conf) throws IOException {
    SequenceFile.Reader extents = null;

    final Path[] files = DistributedCache.getLocalCacheFiles(conf);
    if (files != null && files.length > 0) {
        String extentsname = conf.get("com.lbt.extentsname");
        final LocalFileSystem localfs = FileSystem.getLocal(conf);
        boolean found = false;
        for (Path p : files) {
            if (p.getName().equals(extentsname)) {
                found = true;/*from w w  w . j a v a 2s . c  om*/
                LOG.info("Opening extents file " + p);
                extents = new SequenceFile.Reader(localfs, p, conf);
                break;
            }
        }
        if (!found) {
            LOG.warn("Could not find extents file in local cache named " + extentsname);
        }
    } else if (files == null) {
        throw new RuntimeException("No file paths retrieved from distributed cache");
        // extents = new SequenceFile.Reader(hdpFs, new Path("ceic_extents/part-r-00000"), conf); // TO-DO: fix hard-coding
    }

    if (extents == null) {
        throw new RuntimeException(
                "Could not open extents file. Number of files in the cache: " + files.length);
    }

    return extents;
}