Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream) 

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:com.cloudera.sqoop.manager.MySQLAuthTest.java

License:Apache License

public void doZeroTimestampTest(int testNum, boolean expectSuccess, String connectString)
        throws IOException, SQLException {

    LOG.info("Beginning zero-timestamp test #" + testNum);

    try {//from   w w  w  .  ja  va  2s  .  c  o m
        final String TABLE_NAME = "mysqlTimestampTable" + Integer.toString(testNum);

        // Create a table containing a full-zeros timestamp.
        SqoopOptions options = new SqoopOptions(connectString, TABLE_NAME);
        options.setUsername(AUTH_TEST_USER);
        options.setPassword(AUTH_TEST_PASS);

        manager = new DirectMySQLManager(options);

        Connection connection = null;
        Statement st = null;

        connection = manager.getConnection();
        connection.setAutoCommit(false);
        st = connection.createStatement();

        // create the database table and populate it with data.
        st.executeUpdate("DROP TABLE IF EXISTS " + TABLE_NAME);
        st.executeUpdate("CREATE TABLE " + TABLE_NAME + " (" + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
                + "ts TIMESTAMP NOT NULL)");

        st.executeUpdate("INSERT INTO " + TABLE_NAME + " VALUES(" + "NULL,'0000-00-00 00:00:00.0')");
        connection.commit();
        st.close();
        connection.close();

        // Run the import.
        String[] argv = getArgv(true, false, connectString, TABLE_NAME);
        try {
            runImport(argv);
        } catch (Exception e) {
            if (expectSuccess) {
                // This is unexpected. rethrow.
                throw new RuntimeException(e);
            } else {
                // We expected an error.
                LOG.info("Got exception running import (expected). msg: " + e);
            }
        }

        // Make sure the result file is there.
        Path warehousePath = new Path(this.getWarehouseDir());
        Path tablePath = new Path(warehousePath, TABLE_NAME);
        Path filePath = new Path(tablePath, "part-m-00000");

        File f = new File(filePath.toString());
        if (expectSuccess) {
            assertTrue("Could not find imported data file", f.exists());
            BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
            assertEquals("1,null", r.readLine());
            IOUtils.closeStream(r);
        } else {
            assertFalse("Imported data when expected failure", f.exists());
        }
    } finally {
        LOG.info("Finished zero timestamp test #" + testNum);
    }
}

---

From source file:com.cloudera.sqoop.manager.NetezzaImportManualTest.java

License:Apache License

private void runNetezzaTest(boolean isDirect, String tableName, String[] expectedResults, String... extraArgs)
        throws IOException {

    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, tableName);

    Path filePath;/* ww  w. j a  va  2s  .c  o m*/

    filePath = new Path(tablePath, "part-m-00000");

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import.
        FileListing.recursiveDeleteDir(tableFile);
    }

    String[] argv = getArgv(isDirect, tableName, extraArgs);
    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file : " + f, f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        String[] s = new String[3];
        for (int i = 0; i < s.length; ++i) {
            s[i] = r.readLine();
            LOG.info("Line read from file = " + s[i]);
        }
        Arrays.sort(s);
        for (int i = 0; i < expectedResults.length; ++i) {
            assertEquals(expectedResults[i], s[i]);
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

---

From source file:com.cloudera.sqoop.manager.OracleManagerTest.java

License:Apache License

private void runSecondaryTableTest(String[] expectedResults) throws IOException {
    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, QUALIFIED_SECONDARY_TABLE_NAME);
    Path filePath = new Path(tablePath, "part-m-00000");

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import
        FileListing.recursiveDeleteDir(tableFile);
    }//from  w w w  .j  a  va2 s.c om

    String[] argv = getArgv(QUALIFIED_SECONDARY_TABLE_NAME);

    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file", f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        for (String expectedLine : expectedResults) {
            compareRecords(expectedLine, r.readLine());
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

---

From source file:com.cloudera.sqoop.manager.OracleManagerTest.java

License:Apache License

private void runOracleTest(String[] expectedResults) throws IOException {

    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, TABLE_NAME);
    Path filePath = new Path(tablePath, "part-m-00000");

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import.
        FileListing.recursiveDeleteDir(tableFile);
    }//from w ww  .j a  v  a2 s .com

    String[] argv = getArgv();
    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file", f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        for (String expectedLine : expectedResults) {
            compareRecords(expectedLine, r.readLine());
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

---

From source file:com.cloudera.sqoop.manager.PostgresqlImportTest.java

License:Apache License

private void doImportAndVerify(boolean isDirect, String[] expectedResults, String tableName,
        String... extraArgs) throws IOException {

    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, tableName);
    Path filePath = new Path(tablePath, "part-m-00000");

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import.
        FileListing.recursiveDeleteDir(tableFile);
    }/*w  w  w  . j a va  2  s .c om*/

    String[] argv = getArgv(isDirect, tableName, extraArgs);
    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file, " + f, f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        for (String expectedLine : expectedResults) {
            assertEquals(expectedLine, r.readLine());
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

---

From source file:com.cloudera.sqoop.manager.PostgresqlTest.java

License:Apache License

private void doImportAndVerify(boolean isDirect, String[] expectedResults, String tableName,
        String... extraArgs) throws IOException {

    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, tableName);

    Path filePath;//w  ww . j  a  v a 2  s.  c  o m
    if (isDirect) {
        filePath = new Path(tablePath, "data-00000");
    } else {
        filePath = new Path(tablePath, "part-m-00000");
    }

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import.
        FileListing.recursiveDeleteDir(tableFile);
    }

    String[] argv = getArgv(isDirect, tableName, extraArgs);
    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file, " + f, f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        for (String expectedLine : expectedResults) {
            assertEquals(expectedLine, r.readLine());
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

---

From source file:com.cloudera.sqoop.manager.SQLServerManagerImportManualTest.java

License:Apache License

private void doImportAndVerify(String tableName, String[] expectedResults, String... extraArgs)
        throws IOException {

    Path warehousePath = new Path(this.getWarehouseDir());
    Path tablePath = new Path(warehousePath, tableName);
    Path filePath = new Path(tablePath, "part-m-00000");

    File tableFile = new File(tablePath.toString());
    if (tableFile.exists() && tableFile.isDirectory()) {
        // remove the directory before running the import.
        FileListing.recursiveDeleteDir(tableFile);
    }//from   w w  w  .ja  v  a2s .  c  o m

    String[] argv = getArgv(tableName, extraArgs);
    try {
        runImport(argv);
    } catch (IOException ioe) {
        LOG.error("Got IOException during import: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    }

    File f = new File(filePath.toString());
    assertTrue("Could not find imported data file", f.exists());
    BufferedReader r = null;
    try {
        // Read through the file and make sure it's all there.
        r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
        for (String expectedLine : expectedResults) {
            assertEquals(expectedLine, r.readLine());
        }
    } catch (IOException ioe) {
        LOG.error("Got IOException verifying results: " + ioe.toString());
        ioe.printStackTrace();
        fail(ioe.toString());
    } finally {
        IOUtils.closeStream(r);
    }
}

---

From source file:com.cloudera.sqoop.TestAllTables.java

License:Apache License

public void testMultiTableImport() throws IOException {
    String[] argv = getArgv(true, null);
    runImport(new ImportAllTablesTool(), argv);

    Path warehousePath = new Path(this.getWarehouseDir());
    int i = 0;// w ww .ja v a  2 s  .com
    for (String tableName : this.tableNames) {
        Path tablePath = new Path(warehousePath, tableName);
        Path filePath = new Path(tablePath, "part-m-00000");

        // dequeue the expected value for this table. This
        // list has the same order as the tableNames list.
        String expectedVal = Integer.toString(i++) + "," + this.expectedStrings.get(0);
        this.expectedStrings.remove(0);

        BufferedReader reader = null;
        if (!isOnPhysicalCluster()) {
            reader = new BufferedReader(
                    new InputStreamReader(new FileInputStream(new File(filePath.toString()))));
        } else {
            FileSystem dfs = FileSystem.get(getConf());
            FSDataInputStream dis = dfs.open(filePath);
            reader = new BufferedReader(new InputStreamReader(dis));
        }
        try {
            String line = reader.readLine();
            assertEquals("Table " + tableName + " expected a different string", expectedVal, line);
        } finally {
            IOUtils.closeStream(reader);
        }
    }
}

---

From source file:com.cloudera.sqoop.TestAllTables.java

License:Apache License

public void testMultiTableImportWithExclude() throws IOException {
    String exclude = this.tableNames.get(0);
    String[] argv = getArgv(true, new String[] { exclude });
    runImport(new ImportAllTablesTool(), argv);

    Path warehousePath = new Path(this.getWarehouseDir());
    int i = 0;/*from   w  w w .j a v  a 2s .  c  o m*/
    for (String tableName : this.tableNames) {
        Path tablePath = new Path(warehousePath, tableName);
        Path filePath = new Path(tablePath, "part-m-00000");

        // dequeue the expected value for this table. This
        // list has the same order as the tableNames list.
        String expectedVal = Integer.toString(i++) + "," + this.expectedStrings.get(0);
        this.expectedStrings.remove(0);

        BufferedReader reader = null;
        if (!isOnPhysicalCluster()) {
            reader = new BufferedReader(
                    new InputStreamReader(new FileInputStream(new File(filePath.toString()))));
        } else {
            FSDataInputStream dis;
            FileSystem dfs = FileSystem.get(getConf());
            if (tableName.equals(exclude)) {
                try {
                    dis = dfs.open(filePath);
                    assertFalse(true);
                } catch (FileNotFoundException e) {
                    // Success
                    continue;
                }
            } else {
                dis = dfs.open(filePath);
            }
            reader = new BufferedReader(new InputStreamReader(dis));
        }
        try {
            String line = reader.readLine();
            assertEquals("Table " + tableName + " expected a different string", expectedVal, line);
        } finally {
            IOUtils.closeStream(reader);
        }
    }
}

---

From source file:com.cloudera.sqoop.TestBoundaryQuery.java

License:Apache License

public void runQueryTest(String query, boolean tableImport, int numExpectedResults, int expectedSum,
        String targetDir, String... extraArgs) throws IOException {

    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, tableImport, query, targetDir, extraArgs);
    runImport(argv);//from www . ja v a  2s .c o  m
    try {
        SqoopOptions opts = new ImportTool()
                .parseArguments(getArgv(false, tableImport, query, targetDir, extraArgs), null, null, true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        if (reader.next(key) == null) {
            fail("Empty SequenceFile during import");
        }

        // make sure that the value we think should be at the top, is.
        reader.getCurrentValue(val);

        // We know that these values are two ints separated by a ',' character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually.  Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all the
        // results from the db into the file.
        int curSum = getFirstInt(val.toString());
        int totalResults = 1;

        // now sum up everything else in the file.
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
            totalResults++;
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
        assertEquals("Incorrect number of results for query", numExpectedResults, totalResults);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »