Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream) 

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:org.apache.sqoop.manager.sqlserver.SQLServerQueryManualTest.java

License:Apache License

public void runQueryTest(String query, String firstValStr, int numExpectedResults, int expectedSum,
        String targetDir) throws IOException {

    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, query, targetDir, false);
    runImport(argv);/*from w  w  w. j a  v  a 2s  . com*/
    try {
        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, query, targetDir, false), null, null,
                true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        if (reader.next(key) == null) {
            fail("Empty SequenceFile during import");
        }

        // make sure that the value we think should be at the top, is.
        reader.getCurrentValue(val);
        assertEquals("Invalid ordering within sorted SeqFile", firstValStr, val.toString());

        // We know that these values are two ints separated by a ','
        // character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually. Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all
        // the
        // results from the db into the file.
        int curSum = getFirstInt(val.toString());
        int totalResults = 1;

        // now sum up everything else in the file.
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
            totalResults++;
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
        assertEquals("Incorrect number of results for query", numExpectedResults, totalResults);
    } catch (InvalidOptionsException ioe) {
        LOG.error(StringUtils.stringifyException(ioe));
        fail(ioe.toString());
    } catch (ParseException pe) {
        LOG.error(StringUtils.stringifyException(pe));
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

---

From source file:org.apache.sqoop.manager.sqlserver.SQLServerSplitByManualTest.java

License:Apache License

public void runSplitByTest(String splitByCol, int expectedSum) throws IOException {

    String[] columns = new String[] { "L_ORDERKEY", "L_PARTKEY", "L_SUPPKEY", "L_LINENUMBER", "L_QUANTITY",
            "L_EXTENDEDPRICE", "L_DISCOUNT", "L_TAX", "L_RETURNFLAG", "L_LINESTATUS", "L_SHIPDATE",
            "L_COMMITDATE", "L_RECEIPTDATE", "L_SHIPINSTRUCT", "L_SHIPMODE", "L_COMMENT", };
    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, columns, splitByCol);
    runImport(argv);/*  w  ww  .  j av a  2s.c  o m*/
    try {
        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, columns, splitByCol), null, null,
                true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();
        LOG.debug("Got jar from import job: " + jarFileName);

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        // We know that these values are two ints separated by a ','
        // character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually. Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all
        // the
        // results from the db into the file.

        // Sum up everything in the file.
        int curSum = 0;
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
        }
        System.out.println("Sum : e,c" + expectedSum + " : " + curSum);
        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
    } catch (InvalidOptionsException ioe) {
        LOG.error(StringUtils.stringifyException(ioe));
        fail(ioe.toString());
    } catch (ParseException pe) {
        LOG.error(StringUtils.stringifyException(pe));
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

---

From source file:org.apache.sqoop.manager.sqlserver.SQLServerWhereManualTest.java

License:Apache License

public void runWhereTest(String whereClause, String firstValStr, int numExpectedResults, int expectedSum)
        throws IOException {

    String[] columns = MSSQLTestUtils.getColumns();
    ClassLoader prevClassLoader = null;
    SequenceFile.Reader reader = null;

    String[] argv = getArgv(true, columns, whereClause);
    runImport(argv);/* ww w  .  ja v  a  2s.c o m*/
    try {
        String username = MSSQLTestUtils.getDBUserName();
        String password = MSSQLTestUtils.getDBPassWord();

        SqoopOptions opts = new ImportTool().parseArguments(getArgv(false, columns, whereClause), null, null,
                true);
        opts.setUsername(username);
        opts.setPassword(password);
        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, getTableName());

        reader = SeqFileReader.getSeqFileReader(getDataFilePath().toString());

        // here we can actually instantiate (k, v) pairs.
        Configuration conf = new Configuration();
        Object key = ReflectionUtils.newInstance(reader.getKeyClass(), conf);
        Object val = ReflectionUtils.newInstance(reader.getValueClass(), conf);

        if (reader.next(key) == null) {
            fail("Empty SequenceFile during import");
        }

        // make sure that the value we think should be at the top, is.
        reader.getCurrentValue(val);
        assertEquals("Invalid ordering within sorted SeqFile", firstValStr, val.toString());

        // We know that these values are two ints separated by a ','
        // character.
        // Since this is all dynamic, though, we don't want to actually link
        // against the class and use its methods. So we just parse this back
        // into int fields manually. Sum them up and ensure that we get the
        // expected total for the first column, to verify that we got all
        // the
        // results from the db into the file.
        int curSum = getFirstInt(val.toString());
        int totalResults = 1;

        // now sum up everything else in the file.
        while (reader.next(key) != null) {
            reader.getCurrentValue(val);
            curSum += getFirstInt(val.toString());
            totalResults++;
        }

        assertEquals("Total sum of first db column mismatch", expectedSum, curSum);
        assertEquals("Incorrect number of results for query", numExpectedResults, totalResults);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        IOUtils.closeStream(reader);

        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

---

From source file:org.apache.sqoop.TestAutoResetMapper.java

License:Apache License

public void testMultiTableImportWithAutoMapperReset() throws IOException {

    String[] argv = getArgv();/* w  w w . j a  v a 2  s. co m*/
    runImport(new ImportAllTablesTool(), argv);

    Path warehousePath = new Path(this.getWarehouseDir());
    FileSystem fs = FileSystem.get(getConf());
    for (int i = 0; i < this.tableNames.size(); ++i) {
        String tableName = this.tableNames.get(i);
        LOG.debug("Validating import of " + tableName);
        Path tablePath = new Path(warehousePath, tableName);
        int numPartFiles = 0;
        List<String> importedData = new ArrayList<String>();
        // We expect utmost 2 files
        for (int m = 0; m < 2; ++m) {
            Path filePath = new Path(tablePath, "part-m-0000" + Integer.toString(m));
            if (fs.exists(filePath)) {
                ++numPartFiles;
                LOG.debug("Reading imported file " + filePath);
                BufferedReader reader = null;
                if (!isOnPhysicalCluster()) {
                    reader = new BufferedReader(
                            new InputStreamReader(new FileInputStream(new File(filePath.toString()))));
                } else {
                    FileSystem dfs = FileSystem.get(getConf());
                    FSDataInputStream dis = dfs.open(filePath);
                    reader = new BufferedReader(new InputStreamReader(dis));
                }
                String line = null;
                try {
                    while ((line = reader.readLine()) != null) {
                        importedData.add(line);
                    }
                } finally {
                    IOUtils.closeStream(reader);
                }
            }
        }
        assertEquals("Table " + tableName + " expected a different number of part files", expectedPartFiles[i],
                numPartFiles);
        for (int k = 0; k < this.expectedStrings[i].length; ++k) {
            assertEquals("Table " + tableName + "expected a different string", expectedStrings[i][k],
                    importedData.get(k));
        }
    }
}

---

From source file:org.apache.tajo.pullserver.TajoPullServerService.java

License:Apache License

@Override
public void serviceInit(Configuration conf) throws Exception {
    if (!(conf instanceof TajoConf)) {
        throw new IllegalArgumentException("Configuration must be a TajoConf instance");
    }/*from w  w w.j  ava 2s . co m*/

    ServerBootstrap bootstrap = selector.clone();
    TajoConf tajoConf = (TajoConf) conf;
    try {
        channelInitializer = new HttpChannelInitializer(tajoConf);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    bootstrap.childHandler(channelInitializer).channel(NioServerSocketChannel.class);

    port = conf.getInt(ConfVars.PULLSERVER_PORT.varname, ConfVars.PULLSERVER_PORT.defaultIntVal);
    ChannelFuture future = bootstrap.bind(new InetSocketAddress(port))
            .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE).syncUninterruptibly();

    accepted.add(future.channel());
    port = ((InetSocketAddress) future.channel().localAddress()).getPort();
    conf.set(ConfVars.PULLSERVER_PORT.varname, Integer.toString(port));
    LOG.info(getName() + " listening on port " + port);

    sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY, DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);

    if (STANDALONE) {
        File pullServerPortFile = getPullServerPortFile();
        if (pullServerPortFile.exists()) {
            pullServerPortFile.delete();
        }
        pullServerPortFile.getParentFile().mkdirs();
        LOG.info("Write PullServerPort to " + pullServerPortFile);
        FileOutputStream out = null;
        try {
            out = new FileOutputStream(pullServerPortFile);
            out.write(("" + port).getBytes());
        } catch (Exception e) {
            LOG.fatal("PullServer exists cause can't write PullServer port to " + pullServerPortFile + ", "
                    + e.getMessage(), e);
            System.exit(-1);
        } finally {
            IOUtils.closeStream(out);
        }
    }
    super.serviceInit(conf);
    LOG.info("TajoPullServerService started: port=" + port);
}

---

From source file:org.apache.tajo.pullserver.TajoPullServerService.java

License:Apache License

public static int readPullServerPort() {
    FileInputStream in = null;//from www .  j av  a  2s.com
    try {
        File pullServerPortFile = getPullServerPortFile();

        if (!pullServerPortFile.exists() || pullServerPortFile.isDirectory()) {
            return -1;
        }
        in = new FileInputStream(pullServerPortFile);
        byte[] buf = new byte[1024];
        int readBytes = in.read(buf);
        return Integer.parseInt(new String(buf, 0, readBytes));
    } catch (IOException e) {
        LOG.fatal(e.getMessage(), e);
        return -1;
    } finally {
        IOUtils.closeStream(in);
    }
}

---

From source file:org.apache.tajo.rule.base.CheckHadoopRuntimeVersionRule.java

License:Apache License

public CheckHadoopRuntimeVersionRule() {
    InputStream is = ClassLoader.getSystemResourceAsStream("common-version-info.properties");
    versionInfo = new Properties();
    try {//from   www  . j a v a  2s . com
        versionInfo.load(is);
    } catch (IOException e) {
        LOG.warn(e.getMessage(), e);
    } finally {
        IOUtils.closeStream(is);
    }
}

---

From source file:org.apache.tajo.storage.avro.AvroUtil.java

License:Apache License

public static Schema getAvroSchemaFromHttp(String schemaURL) throws IOException {
    InputStream inputStream = new URL(schemaURL).openStream();

    try {//from w w w  .  j av a2 s.c  om
        return new Schema.Parser().parse(inputStream);
    } finally {
        IOUtils.closeStream(inputStream);
    }
}

---

From source file:org.apache.tajo.storage.avro.AvroUtil.java

License:Apache License

public static Schema getAvroSchemaFromFileSystem(String schemaURL, Configuration conf) throws IOException {
    Path schemaPath = new Path(schemaURL);
    FileSystem fs = schemaPath.getFileSystem(conf);
    FSDataInputStream inputStream = fs.open(schemaPath);

    try {//from ww  w  .  j  a  v  a2  s.  co  m
        return new Schema.Parser().parse(inputStream);
    } finally {
        IOUtils.closeStream(inputStream);
    }
}

---

From source file:org.apache.tajo.storage.TestLineReader.java

License:Apache License

@Test
public void testCRLFLine() throws IOException {
    TajoConf conf = new TajoConf();
    Path testFile = new Path(CommonTestingUtil.getTestDir(TEST_PATH), "testCRLFLineText.txt");

    FileSystem fs = testFile.getFileSystem(conf);
    FSDataOutputStream outputStream = fs.create(testFile, true);
    outputStream.write("0\r\n1\r\n".getBytes());
    outputStream.flush();// w  w  w. j a va 2 s  .  c o  m
    IOUtils.closeStream(outputStream);

    ByteBufInputChannel channel = new ByteBufInputChannel(fs.open(testFile));
    ByteBufLineReader reader = new ByteBufLineReader(channel, BufferPool.directBuffer(2));
    FileStatus status = fs.getFileStatus(testFile);

    long totalRead = 0;
    int i = 0;
    AtomicInteger bytes = new AtomicInteger();
    for (;;) {
        ByteBuf buf = reader.readLineBuf(bytes);
        totalRead += bytes.get();
        if (buf == null)
            break;
        String row = buf.toString(Charset.defaultCharset());
        assertEquals(i, Integer.parseInt(row));
        i++;
    }
    IOUtils.cleanup(null, reader);
    assertEquals(status.getLen(), totalRead);
    assertEquals(status.getLen(), reader.readBytes());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »