Example usage for org.apache.hadoop.io IOUtils copyBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils copyBytes.

Prototype

public static void copyBytes(InputStream in, OutputStream out, Configuration conf) throws IOException 

Source Link

Document

Copies from one stream to another.

Usage

From source file:TestCodec.java

License:Open Source License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    DefaultCodec codec = new DefaultCodec();
    codec.setConf(conf);//from   w  ww  .  ja  v  a 2 s.  co  m
    DataOutputBuffer chunksWriteBuffer = new DataOutputBuffer();
    CompressionOutputStream compressionOutputStream = codec.createOutputStream(chunksWriteBuffer);

    DataInputBuffer chunkReadBuffer = new DataInputBuffer();
    CompressionInputStream compressionInputStream = codec.createInputStream(chunkReadBuffer);
    String str = "laksjldfkjalskdjfl;aksjdflkajsldkfjalksjdflkajlsdkfjlaksjdflka";
    compressionOutputStream.write(str.getBytes());
    compressionOutputStream.finish();
    byte[] data = chunksWriteBuffer.getData();
    System.out.println(str.length());
    System.out.println(chunksWriteBuffer.getLength());

    chunkReadBuffer.reset(data, chunksWriteBuffer.getLength());

    DataOutputBuffer dob = new DataOutputBuffer();
    IOUtils.copyBytes(compressionInputStream, dob, conf);
    System.out.println(dob.getData());

}

---

From source file:cn.lhfei.hadoop.ch04.FileDecompressor.java

License:Apache License

/**
 * use case: % hadoop FileDecompressor file.gz
 * @param args//from   www.  j a  v  a 2  s.c  o  m
 */
public static void main(String[] args) {
    FileSystem fs = null;
    String uri = args[0];
    Path inputPath = null;
    Configuration conf = new Configuration();
    CompressionCodecFactory factory = null;

    InputStream in = null;
    OutputStream out = null;

    try {
        fs = FileSystem.get(URI.create(uri), conf);
        inputPath = new Path(uri);
        factory = new CompressionCodecFactory(conf);
        CompressionCodec codec = factory.getCodec(inputPath);
        if (codec == null) {
            System.err.println("No codec found for " + uri);
            System.exit(1);
        }

        String outputUri = CompressionCodecFactory.removeSuffix(uri, codec.getDefaultExtension());

        in = codec.createInputStream(fs.open(inputPath));
        out = fs.create(new Path(outputUri));

        IOUtils.copyBytes(in, out, conf);

    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
    }
}

---

From source file:com.asakusafw.thundergate.runtime.cache.CacheStorageTest.java

License:Apache License

/**
 * delete HEAD version./*from   ww  w.  jav a 2  s. co m*/
 * @throws Exception if failed
 */
@Test
public void deleteHead() throws Exception {
    File dir = folder.newFolder("testing");
    dir.delete();
    try (CacheStorage storage = new CacheStorage(new Configuration(), dir.toURI())) {
        Path content = storage.getHeadContents("a");
        try (FSDataOutputStream output = storage.getFileSystem().create(content)) {
            IOUtils.copyBytes(new ByteArrayInputStream("Hello, world".getBytes()), output,
                    storage.getConfiguration());
        }

        assertThat(storage.getFileSystem().exists(storage.getHeadDirectory()), is(true));
        assertThat(storage.getFileSystem().exists(storage.getPatchDirectory()), is(false));

        storage.deleteHead();
        assertThat(storage.getFileSystem().exists(storage.getHeadDirectory()), is(false));
    }
}

---

From source file:com.asakusafw.thundergate.runtime.cache.CacheStorageTest.java

License:Apache License

/**
 * delete PATCH version./*from w w  w  .  j av a  2 s .c om*/
 * @throws Exception if failed
 */
@Test
public void deletePatch() throws Exception {
    File dir = folder.newFolder("testing");
    dir.delete();
    try (CacheStorage storage = new CacheStorage(new Configuration(), dir.toURI())) {
        Path content = storage.getPatchContents("a");
        try (FSDataOutputStream output = storage.getFileSystem().create(content)) {
            IOUtils.copyBytes(new ByteArrayInputStream("Hello, world".getBytes()), output,
                    storage.getConfiguration());
        }

        assertThat(storage.getFileSystem().exists(storage.getPatchDirectory()), is(true));
        assertThat(storage.getFileSystem().exists(storage.getHeadDirectory()), is(false));

        storage.deletePatch();
        assertThat(storage.getFileSystem().exists(storage.getPatchDirectory()), is(false));
    }
}

---

From source file:com.asakusafw.thundergate.runtime.cache.CacheStorageTest.java

License:Apache License

/**
 * delete all.//from ww w.  j ava  2  s.c  o m
 * @throws Exception if failed
 */
@Test
public void deleteAll() throws Exception {
    File dir = folder.newFolder("testing");
    dir.delete();
    try (CacheStorage storage = new CacheStorage(new Configuration(), dir.toURI())) {
        Path headContent = storage.getHeadContents("a");
        try (FSDataOutputStream output = storage.getFileSystem().create(headContent)) {
            IOUtils.copyBytes(new ByteArrayInputStream("Hello, world".getBytes()), output,
                    storage.getConfiguration());
        }
        Path patchContent = storage.getPatchContents("a");
        try (FSDataOutputStream output = storage.getFileSystem().create(patchContent)) {
            IOUtils.copyBytes(new ByteArrayInputStream("Hello, world".getBytes()), output,
                    storage.getConfiguration());
        }

        assertThat(storage.getFileSystem().exists(storage.getHeadDirectory()), is(true));
        assertThat(storage.getFileSystem().exists(storage.getPatchDirectory()), is(true));

        assertThat(storage.deleteAll(), is(true));
        assertThat(storage.getFileSystem().exists(storage.getHeadDirectory()), is(false));
        assertThat(storage.getFileSystem().exists(storage.getPatchDirectory()), is(false));
    }
}

---

From source file:com.cloudera.cdk.data.TestDatasetDescriptor.java

License:Apache License

@Test
public void testSchemaFromHdfs() throws IOException {
    FileSystem fs = getDFS();/*  w ww  . j a va 2s  .  c  o  m*/

    // copy a schema to HDFS
    Path schemaPath = fs.makeQualified(new Path("schema.avsc"));
    FSDataOutputStream out = fs.create(schemaPath);
    IOUtils.copyBytes(USER_SCHEMA_URL.toURL().openStream(), out, fs.getConf());
    out.close();

    // build a schema using the HDFS path and check it's the same
    Schema schema = new DatasetDescriptor.Builder().schemaUri(schemaPath.toUri()).build().getSchema();

    Assert.assertEquals(USER_SCHEMA, schema);
}

---

From source file:com.cloudera.cdk.morphline.hadoop.core.DownloadHdfsFileTest.java

License:Apache License

@Test
public void testBasic() throws IOException {
    String msg = "hello world";

    // setup: copy a file to HDFS to prepare inputFile    
    Path inputFile = fileSystem.makeQualified(new Path(testDirectory, fileName));
    FSDataOutputStream out = fileSystem.create(inputFile);
    IOUtils.copyBytes(new ByteArrayInputStream(msg.getBytes(Charsets.UTF_8)), out, fileSystem.getConf());
    out.close();//from   w ww  .  j a  v  a  2  s  . c om

    File cwd = Files.createTempDir().getAbsoluteFile();
    if (isDir) {
        dst = new File(cwd, testDirectory.getName() + "/" + inputFile.getName());
        inputFile = inputFile.getParent();
    } else {
        dst = new File(cwd, inputFile.getName());
    }
    Assert.assertFalse(dst.exists());
    new File(cwd, fileName).mkdirs(); // will be auto deleted!
    Files.write("wrong msg", new File(new File(cwd, fileName), fileName), Charsets.UTF_8); // will be auto deleted!

    Command morphline = createMorphline("test-morphlines/testDownloadHdfsFile", inputFile, cwd);
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertEquals(msg, Files.toString(dst, Charsets.UTF_8));
    if (isDir) {
        FileUtil.fullyDelete(dst.getParentFile());
    } else {
        FileUtil.fullyDelete(dst);
    }
    Assert.assertTrue(fileSystem.exists(inputFile));
    Assert.assertTrue(FileUtil.fullyDelete(cwd));

    // verify that subsequent calls with same inputFile won't copy the file again (to prevent races)
    morphline = createMorphline("test-morphlines/downloadHdfsFile", inputFile, cwd);
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertFalse(dst.exists());
    Assert.assertTrue(morphline.process(new Record()));
    Assert.assertFalse(dst.exists());
    Assert.assertFalse(cwd.exists());

    Assert.assertTrue(fileSystem.delete(inputFile, true));

    try {
        morphline = createMorphline("test-morphlines/downloadHdfsFile", new Path("nonExistingInputFile"), cwd);
        Assert.fail("failed to detect non-existing input file");
    } catch (MorphlineCompilationException e) {
        Assert.assertTrue(e.getCause() instanceof FileNotFoundException);
    }
    Assert.assertFalse(dst.exists());
}

---

From source file:com.cloudera.llama.am.LlamaServlet.java

License:Apache License

@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    if (req.getPathInfo() == null || req.getPathInfo().equals("/")) {
        resp.setContentType("text/html");
        resp.setStatus(HttpServletResponse.SC_OK);
        IOUtils.copyBytes(cl.getResourceAsStream("llama.html"), resp.getOutputStream(), 1024);
    } else {/*from  ww  w . jav  a2 s  . co  m*/
        resp.setStatus(HttpServletResponse.SC_NOT_FOUND);
    }
}

---

From source file:com.cotdp.hadoop.ZipFileTest.java

License:Apache License

/**
 * Simple utility function to copy files into HDFS
 * //from  w ww  . j  av a2  s .  c  o m
 * @param fs
 * @param name
 * @throws IOException
 */
private void copyFile(FileSystem fs, String name) throws IOException {
    LOG.info("copyFile: " + name);
    InputStream is = this.getClass().getResourceAsStream("/" + name);
    OutputStream os = fs.create(new Path(inputPath, name), true);
    IOUtils.copyBytes(is, os, conf);
    os.close();
    is.close();
}

---

From source file:com.examples.ch02.HdfsReader_Ex_2.java

public int run(String[] args) throws Exception {
    Path inputPath = new Path(args[0]);
    String localOutputPath = args[1];
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    InputStream is = fs.open(inputPath);
    OutputStream os = new BufferedOutputStream(new FileOutputStream(localOutputPath));
    IOUtils.copyBytes(is, os, conf);
    return 0;//from   www  .  j  a  v a 2  s  .  c o  m
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »