Example usage for org.apache.hadoop.io IOUtils closeStream

List of usage examples for org.apache.hadoop.io IOUtils closeStream

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. IOUtils
  6. closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream)

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:com.cloudera.sqoop.TestCompression.java

License:Apache License

public void runSequenceFileCompressionTest(CompressionCodec codec, int expectedNum) throws Exception {

    String[] columns = HsqldbTestServer.getFieldNames();
    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, columns, codec, "--as-sequencefile");
    runImport(argv);/*  ww w .  j  a va2 s  . c om*/
    try {
        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, columns, codec, "--as-sequencefile"),
                null, null, true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();
        LOG.debug("Got jar from import job: " + jarFileName);

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        if (codec == null) {
            codec = new GzipCodec();
        }
        assertTrue("Block compressed", reader.isBlockCompressed());
        assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        // We know that these values are two ints separated by a ',' character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually.  Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all the
        // results from the db into the file.

        // Sum up everything in the file.
        int numLines = 0;
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            numLines++;
        }

        assertEquals(expectedNum, numLines);
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

From source file:com.cloudera.sqoop.TestFreeFormQueryImport.java

License:Apache License

/**
 * Create two tables that share the common id column.  Run free-form query
 * import on the result table that is created by joining the two tables on
 * the id column.//  www  . ja v a 2  s . co m
 */
public void testSimpleJoin() throws IOException {
    tableNames = new ArrayList<String>();

    String[] types1 = { "SMALLINT", };
    String[] vals1 = { "1", };
    String tableName1 = getTableName();
    createTableWithColTypes(types1, vals1);
    tableNames.add(tableName1);

    incrementTableNum();

    String[] types2 = { "SMALLINT", "VARCHAR(32)", };
    String[] vals2 = { "1", "'foo'", };
    String tableName2 = getTableName();
    createTableWithColTypes(types2, vals2);
    tableNames.add(tableName2);

    String query = "SELECT " + tableName1 + "." + getColName(0) + ", " + tableName2 + "." + getColName(1) + " "
            + "FROM " + tableName1 + " JOIN " + tableName2 + " ON (" + tableName1 + "." + getColName(0) + " = "
            + tableName2 + "." + getColName(0) + ") WHERE " + tableName1 + "." + getColName(0)
            + " < 3 AND $CONDITIONS";

    runImport(getArgv(tableName1 + "." + getColName(0), query));

    Path warehousePath = new Path(this.getWarehouseDir());
    Path filePath = new Path(warehousePath, "part-m-00000");
    String expectedVal = "1,foo";

    BufferedReader reader = null;
    if (!isOnPhysicalCluster()) {
        reader = new BufferedReader(new InputStreamReader(new FileInputStream(new File(filePath.toString()))));
    } else {
        FileSystem dfs = FileSystem.get(getConf());
        FSDataInputStream dis = dfs.open(filePath);
        reader = new BufferedReader(new InputStreamReader(dis));
    }
    try {
        String line = reader.readLine();
        assertEquals("QueryResult expected a different string", expectedVal, line);
    } finally {
        IOUtils.closeStream(reader);
    }
}

From source file:com.cloudera.sqoop.TestMultiMaps.java

License:Apache License

public void runMultiMapTest(String splitByCol, int expectedSum) throws IOException {

    String[] columns = HsqldbTestServer.getFieldNames();
    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, columns, splitByCol);
    runImport(argv);/*  w  ww. j  av  a2s .co m*/
    try {
        ImportTool importTool = new ImportTool();
        SqoopOptions opts = importTool.parseArguments(getArgv(false, columns, splitByCol), null, null, true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        List<Path> paths = getDataFilePaths();
        Configuration conf = new Configuration();
        int curSum = 0;

        // We expect multiple files. We need to open all the files and sum up the
        // first column across all of them.
        for (Path p : paths) {
            reader = SeqFileReader.getSeqFileReader(p.toString());

            // here we can actually instantiate (k, v) pairs.
            Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
            Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

            // We know that these values are two ints separated by a ','
            // character.  Since this is all dynamic, though, we don't want to
            // actually link against the class and use its methods. So we just
            // parse this back into int fields manually.  Sum them up and ensure
            // that we get the expected total for the first column, to verify that
            // we got all the results from the db into the file.

            // now sum up everything in the file.
            while (reader.next(key) != null) {
                reader.getCurrentValue(val);
                curSum += getFirstInt(val.toString());
            }

            IOUtils.closeStream(reader);
            reader = null;
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

From source file:com.cloudera.sqoop.TestQuery.java

License:Apache License

public void runQueryTest(String query, String firstValStr, int numExpectedResults, int expectedSum,
        String targetDir) throws IOException {

    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, query, targetDir, false);
    runImport(argv);//ww  w .  j av a2  s  .  com
    try {
        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, query, targetDir, false), null, null,
                true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        if (reader.next(key) == null) {
            fail("Empty SequenceFile during import");
        }

        // make sure that the value we think should be at the top, is.
        reader.getCurrentValue(val);
        assertEquals("Invalid ordering within sorted SeqFile", firstValStr, val.toString());

        // We know that these values are two ints separated by a ',' character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually.  Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all the
        // results from the db into the file.
        int curSum = getFirstInt(val.toString());
        int totalResults = 1;

        // now sum up everything else in the file.
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
            totalResults++;
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
        assertEquals("Incorrect number of results for query", numExpectedResults, totalResults);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

From source file:com.cloudera.sqoop.TestSplitBy.java

License:Apache License

public void runSplitByTest(String splitByCol, int expectedSum) throws IOException {

    String[] columns = HsqldbTestServer.getFieldNames();
    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, columns, splitByCol);
    runImport(argv);/*from  w  ww .j av  a2 s  . co m*/
    try {
        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, columns, splitByCol), null, null,
                true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();
        LOG.debug("Got jar from import job: " + jarFileName);

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        // We know that these values are two ints separated by a ',' character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually.  Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all the
        // results from the db into the file.

        // Sum up everything in the file.
        int curSum = 0;
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

From source file:com.cloudera.sqoop.TestWhere.java

License:Apache License

public void runWhereTest(String whereClause, String firstValStr, int numExpectedResults, int expectedSum)
        throws IOException {

    String[] columns = HsqldbTestServer.getFieldNames();
    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, columns, whereClause);
    runImport(argv);//from   www.  ja  v a 2 s .c  o m
    try {
        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, columns, whereClause), null, null,
                true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        if (reader.next(key) == null) {
            fail("Empty SequenceFile during import");
        }

        // make sure that the value we think should be at the top, is.
        reader.getCurrentValue(val);
        assertEquals("Invalid ordering within sorted SeqFile", firstValStr, val.toString());

        // We know that these values are two ints separated by a ',' character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually.  Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all the
        // results from the db into the file.
        int curSum = getFirstInt(val.toString());
        int totalResults = 1;

        // now sum up everything else in the file.
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
            totalResults++;
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
        assertEquals("Incorrect number of results for query", numExpectedResults, totalResults);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

From source file:com.couchbase.sqoop.manager.CouchbaseManagerTest.java

License:Apache License

private void runCouchbaseTest(HashMap<String, String> expectedMap) throws IOException {
    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, TABLE_NAME);
    Path filePath = new Path(tablePath, "part-m-00000");

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import.
        FileListing.recursiveDeleteDir(tableFile);
    }//from  w  w  w  . j a  v a  2s .c  om

    String[] argv = getArgv();
    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file", f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        String line;
        int records = 0;
        while ((line = r.readLine()) != null) {
            compareRecords(expectedMap, line);
            records++;
        }
        if (records < NUM_RECORDS) {
            fail("Not everything was imported. Got " + records + "/" + NUM_RECORDS + " records.");
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

From source file:com.datatorrent.common.util.AsyncFSStorageAgent.java

License:Apache License

public void copyToHDFS(final int operatorId, final long windowId) throws IOException {
    if (this.localBasePath == null) {
        throw new AssertionError("save() was not called before copyToHDFS");
    }/*ww  w.  j a v a2 s.c  o m*/
    String operatorIdStr = String.valueOf(operatorId);
    File directory = new File(localBasePath, operatorIdStr);
    String window = Long.toHexString(windowId);
    Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
    File srcFile = new File(directory, String.valueOf(windowId));
    FSDataOutputStream stream = null;
    boolean stateSaved = false;
    try {
        // Create the temporary file with OverWrite option to avoid dangling lease issue and avoid exception if file already exists
        stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                Options.CreateOpts.CreateParent.createParent());
        InputStream in = null;
        try {
            in = new FileInputStream(srcFile);
            IOUtils.copyBytes(in, stream, conf, false);
        } finally {
            IOUtils.closeStream(in);
        }
        stateSaved = true;
    } catch (Throwable t) {
        logger.debug("while saving {} {}", operatorId, window, t);
        stateSaved = false;
        throw Throwables.propagate(t);
    } finally {
        try {
            if (stream != null) {
                stream.close();
            }
        } catch (IOException ie) {
            stateSaved = false;
            throw new RuntimeException(ie);
        } finally {
            if (stateSaved) {
                fileContext.rename(lPath,
                        new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
                        Options.Rename.OVERWRITE);
            }
            FileUtil.fullyDelete(srcFile);
        }
    }
}

From source file:com.datatorrent.contrib.hdht.HadoopFilePerformanceTest.java

License:Open Source License

private void writeMapFile() throws Exception {
    Path path = Testfile.MAPFILE.filepath();

    Text key = new Text();
    Text value = new Text();

    long fsMinBlockSize = conf.getLong("dfs.namenode.fs-limits.min-block-size", 0);

    long testBlockSize = (blockSize < fsMinBlockSize) ? fsMinBlockSize : (long) blockSize;

    MapFile.Writer writer = new MapFile.Writer(conf, path, MapFile.Writer.keyClass(key.getClass()),
            MapFile.Writer.valueClass(value.getClass()),
            MapFile.Writer.compression(SequenceFile.CompressionType.NONE),
            SequenceFile.Writer.blockSize(testBlockSize), SequenceFile.Writer.bufferSize((int) testBlockSize));
    for (int i = 0; i < testSize; i++) {
        key.set(getKey(i));//  w w w .ja  va  2s .  c o m
        value.set(getValue());
        writer.append(key, value);
    }
    IOUtils.closeStream(writer);
}

From source file:com.digitalpebble.behemoth.util.CorpusGenerator.java

License:Apache License

public long generate(boolean recurse) throws IOException {
    long result = 0;
    // read from input path
    // create new Content object and add it to the SequenceFile
    Text key = new Text();
    BehemothDocument value = new BehemothDocument();
    SequenceFile.Writer writer = null;
    try {/*  ww w . j  av  a  2s.c  o  m*/
        Configuration conf = getConf();
        FileSystem fs = output.getFileSystem(conf);
        writer = SequenceFile.createWriter(fs, conf, output, key.getClass(), value.getClass());
        PerformanceFileFilter pff = new PerformanceFileFilter(writer, key, value, conf, reporter);
        // iterate on the files in the source dir
        result = processFiles(conf, input, recurse, pff);

    } finally {
        IOUtils.closeStream(writer);
    }
    return result;
}