Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.cloudera.cdk.data.filesystem.TestCSVFileReader.java

License:Apache License

@BeforeClass
public static void createCSVFiles() throws IOException {
    localfs = FileSystem.getLocal(new Configuration());
    csvFile = new Path("target/temp.csv");
    tsvFile = new Path("target/temp.tsv");
    validatorFile = new Path("target/validator.csv");

    FSDataOutputStream out = localfs.create(csvFile, true);
    out.writeBytes(CSV_CONTENT);/*w  ww.j av  a 2s.  c o  m*/
    out.close();

    out = localfs.create(validatorFile, true);
    out.writeBytes(VALIDATOR_CSV_CONTENT);
    out.close();

    out = localfs.create(tsvFile, true);
    out.writeBytes(TSV_CONTENT);
    out.close();
}

From source file:com.cloudera.cdk.data.filesystem.TestFileSystemDatasetReader.java

License:Apache License

@Override
public DatasetReader newReader() throws IOException {
    return new FileSystemDatasetReader<String>(FileSystem.getLocal(new Configuration()),
            new Path(Resources.getResource("data/strings-100.avro").getFile()), STRING_SCHEMA);
}

From source file:com.cloudera.cdk.data.filesystem.TestFileSystemDatasetReader.java

License:Apache License

@Before
public void setUp() throws IOException {
    fileSystem = FileSystem.getLocal(new Configuration());
}

From source file:com.cloudera.cdk.data.MiniDFSTest.java

License:Apache License

@BeforeClass
public static void setupFS() throws IOException {
    if (cluster == null) {
        cluster = new MiniDFSCluster.Builder(new Configuration()).build();
        dfs = cluster.getFileSystem();/*from w  ww . ja va 2  s .  co  m*/
        conf = new Configuration(dfs.getConf());
        lfs = FileSystem.getLocal(conf);
    }
}

From source file:com.cloudera.cdk.examples.data.CreateUserDatasetPojo.java

License:Apache License

@Override
public int run(String[] args) throws IOException {

    // Construct a local filesystem dataset repository rooted at /tmp/data
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path root = new Path("/tmp/data");
    DatasetRepository repo = new FileSystemDatasetRepository(fs, root);

    // Generate an Avro schema from the User class
    Schema schema = ReflectData.get().getSchema(User.class);

    // Create a dataset of users with the Avro schema in the repository
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder().schema(schema).get();
    Dataset users = repo.create("users", descriptor);

    // Get a writer for the dataset and write some users to it
    DatasetWriter<User> writer = users.getWriter();
    try {/*  www .j  av  a2  s.  co  m*/
        writer.open();
        String[] colors = { "green", "blue", "pink", "brown", "yellow" };
        Random rand = new Random();
        for (int i = 0; i < 100; i++) {
            User user = new User();
            user.setUsername("user-" + i);
            user.setCreationDate(System.currentTimeMillis());
            user.setFavoriteColor(colors[rand.nextInt(colors.length)]);
            writer.write(user);
        }
    } finally {
        writer.close();
    }

    return 0;
}

From source file:com.cloudera.cdk.examples.data.DropUserDataset.java

License:Apache License

@Override
public int run(String[] args) throws IOException {

    // Construct a local filesystem dataset repository rooted at /tmp/data
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path root = new Path("/tmp/data");
    DatasetRepository repo = new FileSystemDatasetRepository(fs, root);

    // Drop the users dataset
    boolean success = repo.drop("users");

    return success ? 0 : 1;
}

From source file:com.cloudera.cdk.examples.data.ReadUserDatasetPojo.java

License:Apache License

@Override
public int run(String[] args) throws IOException {

    // Construct a local filesystem dataset repository rooted at /tmp/data
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path root = new Path("/tmp/data");
    DatasetRepository repo = new FileSystemDatasetRepository(fs, root);

    // Get the users dataset
    Dataset users = repo.get("users");

    // Get a reader for the dataset and read all the users
    DatasetReader<User> reader = users.getReader();
    try {/*from   w w  w.  j  a v a 2s  . co  m*/
        reader.open();
        while (reader.hasNext()) {
            User user = reader.read();
            System.out.println(user);
        }
    } finally {
        reader.close();
    }

    return 0;
}

From source file:com.cloudera.cdk.morphline.hadoop.core.MiniDFSTest.java

License:Apache License

@BeforeClass
public static void setupFS() throws IOException {
    final Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).build();
    dfs = cluster.getFileSystem();/*from   w ww  . j a  v  a 2s  . co  m*/
    lfs = FileSystem.getLocal(conf);
}

From source file:com.cloudera.flume.collector.SequenceFileMerger.java

License:Apache License

public static void main(String[] argv) throws IOException {
    if (argv.length < 2) {
        System.out.println("need to specify target output file, and source dir");
        System.exit(-1);/*from  ww w.ja va  2s.  c om*/
    }

    FlumeConfiguration conf = FlumeConfiguration.get();
    Path dst = new Path(argv[0]);
    String src = argv[1];
    FileSystem fs = FileSystem.getLocal(conf);

    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, dst, WriteableEventKey.class,
            WriteableEvent.class);

    long count = 0;
    long fcount = 0;
    File[] files = new File(src).listFiles();
    for (File f : files) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(f.getAbsolutePath()), conf);
        System.out.println(f.getName());
        fcount++;
        boolean hasNext = true;
        while (hasNext) {
            WriteableEventKey k = new WriteableEventKey();
            WriteableEvent e = new WriteableEvent();
            hasNext = reader.next(k, e);
            if (hasNext) {
                writer.append(k, e);
                count++;
            }
        }
        reader.close();
    }

    writer.close();
    System.out.println("Wrote " + count + " entries from " + fcount + " files.");
}

From source file:com.cloudera.flume.handlers.hdfs.SeqfileEventSink.java

License:Apache License

public void open() throws IOException {
    LOG.debug("opening " + f);

    // need absolute file to get full path. (pwd files have not parent file)
    // make directories if necessary
    if (!FileUtil.makeDirs(f.getAbsoluteFile().getParentFile())) {
        throw new IOException(
                "Unable to create directory" + f.getAbsoluteFile().getParentFile() + " for writing");
    }/*from  w w w  .  jav  a2  s.  c  om*/

    FlumeConfiguration conf = FlumeConfiguration.get();
    FileSystem fs = FileSystem.getLocal(conf);

    try {

        if (conf.getWALOutputBuffering()) {
            writer = RawSequenceFileWriter.createWriter(fs, conf, new Path(f.getAbsolutePath()),
                    WriteableEventKey.class, WriteableEvent.class, CompressionType.NONE);

        } else {
            writer = FlushingSequenceFileWriter.createWriter(conf, f, WriteableEventKey.class,
                    WriteableEvent.class);

        }

    } catch (FileNotFoundException fnfe) {
        LOG.error("Possible permissions problem when creating " + f, fnfe);
        throw fnfe;
    }
}