Example usage for org.apache.hadoop.io IOUtils cleanup

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils cleanup.

Prototype

@Deprecated
public static void cleanup(Log log, java.io.Closeable... closeables) 

Source Link

Document

Close the Closeable objects and ignore any Throwable or null pointers.

Usage

From source file:org.apache.tajo.storage.avro.AvroAppender.java

License:Apache License

/**
 * Closes the Appender.
 */
@Override
public void close() throws IOException {
    IOUtils.cleanup(null, dataFileWriter);
}

---

From source file:org.apache.tajo.storage.http.ExampleHttpJsonLineReader.java

License:Apache License

@Override
public void close() throws IOException {
    try {//from   w  w  w .  j av  a 2  s . c  o  m
        IOUtils.cleanup(LOG, lineReader);

        if (connection != null) {
            connection.disconnect();
        }

        is = null;
        lineReader = null;

    } finally {
        CodecPool.returnDecompressor(decompressor);
        decompressor = null;
    }
}

---

From source file:org.apache.tajo.storage.http.ExampleHttpJsonScanner.java

License:Apache License

@Override
public void close() throws IOException {
    try {/*w  w  w.  j  a  v a2  s.c  o  m*/

        if (deserializer != null) {
            deserializer.release();
        }

        if (reader != null) {
            inputStats.setReadBytes(reader.getReadBytes());
            inputStats.setNumRows(recordCount);
        }

    } finally {
        IOUtils.cleanup(LOG, reader);
        outTuple = null;
    }
}

---

From source file:org.apache.tajo.storage.LocalFileInputChannel.java

License:Apache License

@Override
protected void implCloseChannel() throws IOException {
    IOUtils.cleanup(null, channel);
}

---

From source file:org.apache.tajo.storage.parquet.ParquetAppender.java

License:Apache License

/**
 * Closes the Appender.
 */
@Override
public void close() throws IOException {
    IOUtils.cleanup(null, writer);
}

---

From source file:org.apache.tajo.storage.rawfile.DirectRawFileScanner.java

License:Apache License

@Override
public void close() throws IOException {
    if (tableStats != null) {
        tableStats.setReadBytes(fileSize);
        tableStats.setNumRows(recordCount);
    }/* ww  w .  j av a  2s .  c o  m*/
    tupleBuffer.release();
    tupleBuffer = null;
    reader = null;

    IOUtils.cleanup(LOG, channel);
}

---

From source file:org.apache.tajo.storage.TestLineReader.java

License:Apache License

@Test
public void testByteBufLineReaderWithoutTerminating() throws IOException {
    String path = FileUtil.getResourcePath("dataset/testLineText.txt").getFile();
    File file = new File(path);
    String data = FileUtil.readTextFile(file);

    ByteBufInputChannel channel = new ByteBufInputChannel(new FileInputStream(file));
    ByteBufLineReader reader = new ByteBufLineReader(channel);

    long totalRead = 0;
    int i = 0;/*from w ww  .j  av a 2 s . c  o  m*/
    AtomicInteger bytes = new AtomicInteger();
    for (;;) {
        ByteBuf buf = reader.readLineBuf(bytes);
        totalRead += bytes.get();
        if (buf == null)
            break;
        i++;
    }
    IOUtils.cleanup(null, reader);
    assertEquals(file.length(), totalRead);
    assertEquals(file.length(), reader.readBytes());
    assertEquals(data.split("\n").length, i);
}

---

From source file:org.apache.tajo.storage.TestLineReader.java

License:Apache License

@Test
public void testCRLFLine() throws IOException {
    TajoConf conf = new TajoConf();
    Path testFile = new Path(CommonTestingUtil.getTestDir(TEST_PATH), "testCRLFLineText.txt");

    FileSystem fs = testFile.getFileSystem(conf);
    FSDataOutputStream outputStream = fs.create(testFile, true);
    outputStream.write("0\r\n1\r\n".getBytes());
    outputStream.flush();/*from   w  w w.  j  a  v a  2s .  co  m*/
    IOUtils.closeStream(outputStream);

    ByteBufInputChannel channel = new ByteBufInputChannel(fs.open(testFile));
    ByteBufLineReader reader = new ByteBufLineReader(channel, BufferPool.directBuffer(2));
    FileStatus status = fs.getFileStatus(testFile);

    long totalRead = 0;
    int i = 0;
    AtomicInteger bytes = new AtomicInteger();
    for (;;) {
        ByteBuf buf = reader.readLineBuf(bytes);
        totalRead += bytes.get();
        if (buf == null)
            break;
        String row = buf.toString(Charset.defaultCharset());
        assertEquals(i, Integer.parseInt(row));
        i++;
    }
    IOUtils.cleanup(null, reader);
    assertEquals(status.getLen(), totalRead);
    assertEquals(status.getLen(), reader.readBytes());
}

---

From source file:org.apache.tajo.util.FileUtil.java

License:Apache License

public static String readTextFile(File file) throws IOException {
    StringBuilder fileData = new StringBuilder(1000);
    BufferedReader reader = new BufferedReader(new FileReader(file));
    char[] buf = new char[1024];
    int numRead;//from   w  ww .jav a 2 s  .c om
    try {
        while ((numRead = reader.read(buf)) != -1) {
            String readData = String.valueOf(buf, 0, numRead);
            fileData.append(readData);
            buf = new char[1024];
        }
    } finally {
        IOUtils.cleanup(null, reader);
    }
    return fileData.toString();
}

---

From source file:org.apache.tajo.util.history.HistoryReader.java

License:Apache License

public List<QueryInfo> getQueries(String keyword) throws IOException {
    List<QueryInfo> queryInfos = new ArrayList<QueryInfo>();

    FileSystem fs = HistoryWriter.getNonCrcFileSystem(historyParentPath, tajoConf);
    try {/* w w w.ja va  2  s  .  c  om*/
        if (!fs.exists(historyParentPath)) {
            return queryInfos;
        }
    } catch (Throwable e) {
        return queryInfos;
    }

    FileStatus[] files = fs.listStatus(historyParentPath);
    if (files == null || files.length == 0) {
        return queryInfos;
    }

    for (FileStatus eachDateFile : files) {
        Path queryListPath = new Path(eachDateFile.getPath(), HistoryWriter.QUERY_LIST);
        if (eachDateFile.isFile() || !fs.exists(queryListPath)) {
            continue;
        }

        FileStatus[] dateFiles = fs.listStatus(queryListPath);
        if (dateFiles == null || dateFiles.length == 0) {
            continue;
        }

        for (FileStatus eachFile : dateFiles) {
            Path path = eachFile.getPath();
            if (eachFile.isDirectory() || !path.getName().endsWith(HistoryWriter.HISTORY_FILE_POSTFIX)) {
                continue;
            }

            FSDataInputStream in = null;
            try {
                in = fs.open(path);

                byte[] buf = new byte[100 * 1024];
                while (true) {
                    int length = in.readInt();
                    if (length > buf.length) {
                        buf = new byte[length];
                    }
                    in.readFully(buf, 0, length);
                    String queryInfoJson = new String(buf, 0, length, Bytes.UTF8_CHARSET);
                    QueryInfo queryInfo = QueryInfo.fromJson(queryInfoJson);
                    if (keyword != null) {
                        if (queryInfo.getSql().indexOf(keyword) >= 0) {
                            queryInfos.add(queryInfo);
                        }
                    } else {
                        queryInfos.add(queryInfo);
                    }
                }
            } catch (EOFException e) {
            } catch (Throwable e) {
                LOG.warn("Reading error:" + path + ", " + e.getMessage());
            } finally {
                IOUtils.cleanup(LOG, in);
            }
        }
    }

    Collections.sort(queryInfos, new Comparator<QueryInfo>() {
        @Override
        public int compare(QueryInfo query1, QueryInfo query2) {
            return query2.getQueryIdStr().toString().compareTo(query1.getQueryIdStr().toString());
        }
    });

    return queryInfos;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »