Example usage for org.apache.spark.memory MemoryMode OFF_HEAP

Introduction

In this page you can find the example usage for org.apache.spark.memory MemoryMode OFF_HEAP.

Prototype

MemoryMode OFF_HEAP

To view the source code for org.apache.spark.memory MemoryMode OFF_HEAP.

Click Source Link

Usage

From source file:org.apache.carbondata.stream.CarbonStreamRecordReader.java

License:Apache License

private boolean scanBlockletAndFillVector(BlockletHeader header) throws IOException {
    // if filter is null and output projection is empty, use the row number of blocklet header
    if (skipScanData) {
        int rowNums = header.getBlocklet_info().getNum_rows();
        vectorProxy = new CarbonVectorProxy(MemoryMode.OFF_HEAP, outputSchema, rowNums, false);
        vectorProxy.setNumRows(rowNums);
        input.skipBlockletData(true);/*from  w w  w. j  a  v a 2s.com*/
        return rowNums > 0;
    }

    input.readBlockletData(header);
    vectorProxy = new CarbonVectorProxy(MemoryMode.OFF_HEAP, outputSchema, input.getRowNums(), false);
    int rowNum = 0;
    if (null == filter) {
        while (input.hasNext()) {
            readRowFromStream();
            putRowToColumnBatch(rowNum++);
        }
    } else {
        try {
            while (input.hasNext()) {
                readRowFromStream();
                if (filter.applyFilter(filterRow, carbonTable.getDimensionOrdinalMax())) {
                    putRowToColumnBatch(rowNum++);
                }
            }
        } catch (FilterUnsupportedException e) {
            throw new IOException("Failed to filter row in vector reader", e);
        }
    }
    vectorProxy.setNumRows(rowNum);
    return rowNum > 0;
}

---

From source file:org.apache.carbondata.streaming.CarbonStreamRecordReader.java

License:Apache License

private boolean scanBlockletAndFillVector(BlockletHeader header) throws IOException {
    // if filter is null and output projection is empty, use the row number of blocklet header
    if (skipScanData) {
        int rowNums = header.getBlocklet_info().getNum_rows();
        columnarBatch = ColumnarBatch.allocate(outputSchema, MemoryMode.OFF_HEAP, rowNums);
        columnarBatch.setNumRows(rowNums);
        input.skipBlockletData(true);/*from   w ww.  j  av  a2s .  com*/
        return rowNums > 0;
    }

    input.readBlockletData(header);
    columnarBatch = ColumnarBatch.allocate(outputSchema, MemoryMode.OFF_HEAP, input.getRowNums());
    int rowNum = 0;
    if (null == filter) {
        while (input.hasNext()) {
            readRowFromStream();
            putRowToColumnBatch(rowNum++);
        }
    } else {
        try {
            while (input.hasNext()) {
                readRowFromStream();
                if (filter.applyFilter(filterRow, carbonTable.getDimensionOrdinalMax())) {
                    putRowToColumnBatch(rowNum++);
                }
            }
        } catch (FilterUnsupportedException e) {
            throw new IOException("Failed to filter row in vector reader", e);
        }
    }
    columnarBatch.setNumRows(rowNum);
    return rowNum > 0;
}

---