Example usage for org.apache.hadoop.hdfs DistributedFileSystem copyFromLocalFile

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DistributedFileSystem copyFromLocalFile.

Prototype

public void copyFromLocalFile(Path src, Path dst) throws IOException 

Source Link

Document

The src file is on the local disk.

Usage

From source file:org.dkpro.bigdata.io.hadoop.HdfsResourceLoaderLocatorTest.java

License:Apache License

@Test
public void testDKProResourceLoading() throws Exception {
    String hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";

    DistributedFileSystem fs = hdfsCluster.getFileSystem();
    fs.mkdirs(new Path("/user/test"));
    fs.copyFromLocalFile(new Path("src/test/resources/hdfsLocator/one.data"), new Path("/user/test/"));
    fs.copyFromLocalFile(new Path("src/test/resources/hdfsLocator/two.data"), new Path("/user/test/"));

    ExternalResourceDescription hdfsResource = ExternalResourceFactory.createExternalResourceDescription(
            HdfsResourceLoaderLocator.class, HdfsResourceLoaderLocator.PARAM_FILESYSTEM, hdfsURI);

    CollectionReader reader = CollectionReaderFactory.createReader(TextReader.class,
            TextReader.KEY_RESOURCE_RESOLVER, hdfsResource, TextReader.PARAM_SOURCE_LOCATION, "hdfs:/user/test",
            TextReader.PARAM_PATTERNS, "*.data");

    List<String> documents = readDocuments(reader);

    assertEquals(2, documents.size());/*  ww  w.  ja  va  2  s .co m*/
    assertTrue(documents.get(0).equals("Text of file one."));
    assertTrue(documents.get(1).equals("Text of file two."));

}

---