List of usage examples for org.apache.hadoop.fs FileSystem getContentSummary
public ContentSummary getContentSummary(Path f) throws IOException
From source file:org.apache.sysml.runtime.util.MapReduceTool.java
License:Apache License
/** * Returns the size of a file or directory on hdfs in bytes. * //from w ww . j a va2s. c o m * @param path file system path * @return file size * @throws IOException if IOException occurs */ public static long getFilesizeOnHDFS(Path path) throws IOException { FileSystem fs = IOUtilFunctions.getFileSystem(path); long ret = 0; //in bytes if (fs.isDirectory(path)) ret = fs.getContentSummary(path).getLength(); else ret = fs.getFileStatus(path).getLen(); //note: filestatus would return 0 on directories return ret; }
From source file:org.apache.tajo.catalog.store.HCatalogStore.java
License:Apache License
@Override public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName) throws CatalogException { org.apache.hadoop.hive.ql.metadata.Table table = null; HCatalogStoreClientPool.HCatalogStoreClient client = null; Path path = null;/*from w w w. j a va 2 s . c om*/ CatalogProtos.StoreType storeType = null; org.apache.tajo.catalog.Schema schema = null; KeyValueSet options = null; TableStats stats = null; PartitionMethodDesc partitions = null; ////////////////////////////////// // set tajo table schema. ////////////////////////////////// try { // get hive table schema try { client = clientPool.getClient(); table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); path = table.getPath(); } catch (NoSuchObjectException nsoe) { throw new CatalogException("Table not found. - tableName:" + tableName, nsoe); } catch (Exception e) { throw new CatalogException(e); } // convert hcatalog field schema into tajo field schema. schema = new org.apache.tajo.catalog.Schema(); HCatSchema tableSchema = null; try { tableSchema = HCatUtil.getTableSchemaWithPtnCols(table); } catch (IOException ioe) { throw new CatalogException("Fail to get table schema. - tableName:" + tableName, ioe); } List<HCatFieldSchema> fieldSchemaList = tableSchema.getFields(); boolean isPartitionKey = false; for (HCatFieldSchema eachField : fieldSchemaList) { isPartitionKey = false; if (table.getPartitionKeys() != null) { for (FieldSchema partitionKey : table.getPartitionKeys()) { if (partitionKey.getName().equals(eachField.getName())) { isPartitionKey = true; } } } if (!isPartitionKey) { String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(eachField.getType().toString()); schema.addColumn(fieldName, dataType); } } // validate field schema. try { HCatalogUtil.validateHCatTableAndTajoSchema(tableSchema); } catch (Exception e) { throw new CatalogException("HCatalog cannot support schema. - schema:" + tableSchema.toString(), e); } stats = new TableStats(); options = new KeyValueSet(); options.putAll(table.getParameters()); options.remove("EXTERNAL"); Properties properties = table.getMetadata(); if (properties != null) { // set field delimiter String fieldDelimiter = "", nullFormat = ""; if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) { fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM); } else { // if hive table used default row format delimiter, Properties doesn't have it. // So, Tajo must set as follows: fieldDelimiter = "\u0001"; } // set null format if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) { nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT); } else { nullFormat = "\\N"; } options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT); // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat)); if (storeType.equals(CatalogProtos.StoreType.TEXTFILE)) { options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) { options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (ColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE)) { options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinarySerDe.class.getName().equals(serde)) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (LazySimpleSerDe.class.getName().equals(serde)) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } // set data size long totalSize = 0; if (properties.getProperty("totalSize") != null) { totalSize = Long.parseLong(properties.getProperty("totalSize")); } else { try { FileSystem fs = path.getFileSystem(conf); if (fs.exists(path)) { totalSize = fs.getContentSummary(path).getLength(); } } catch (IOException ioe) { throw new CatalogException("Fail to get path. - path:" + path.toString(), ioe); } } stats.setNumBytes(totalSize); } // set partition keys List<FieldSchema> partitionKeys = table.getPartitionKeys(); if (null != partitionKeys) { org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); StringBuilder sb = new StringBuilder(); if (partitionKeys.size() > 0) { for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HCatalogUtil .getTajoFieldType(fieldSchema.getType().toString()); String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.addColumn(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); } sb.append(fieldSchema.getName()); } partitions = new PartitionMethodDesc(databaseName, tableName, PartitionType.COLUMN, sb.toString(), expressionSchema); } } } finally { if (client != null) client.release(); } TableMeta meta = new TableMeta(storeType, options); TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { tableDesc.setExternal(true); } if (stats != null) { tableDesc.setStats(stats); } if (partitions != null) { tableDesc.setPartitionMethod(partitions); } return tableDesc.getProto(); }
From source file:org.apache.tajo.catalog.store.HiveCatalogStore.java
License:Apache License
@Override public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName) throws CatalogException { org.apache.hadoop.hive.ql.metadata.Table table = null; HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; Path path = null;//from www . j a v a2s.com CatalogProtos.StoreType storeType = null; org.apache.tajo.catalog.Schema schema = null; KeyValueSet options = null; TableStats stats = null; PartitionMethodDesc partitions = null; ////////////////////////////////// // set tajo table schema. ////////////////////////////////// try { // get hive table schema try { client = clientPool.getClient(); table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); path = table.getPath(); } catch (NoSuchObjectException nsoe) { throw new CatalogException("Table not found. - tableName:" + tableName, nsoe); } catch (Exception e) { throw new CatalogException(e); } // convert HiveCatalogStore field schema into tajo field schema. schema = new org.apache.tajo.catalog.Schema(); List<FieldSchema> fieldSchemaList = table.getCols(); boolean isPartitionKey = false; for (FieldSchema eachField : fieldSchemaList) { isPartitionKey = false; if (table.getPartitionKeys() != null) { for (FieldSchema partitionKey : table.getPartitionKeys()) { if (partitionKey.getName().equals(eachField.getName())) { isPartitionKey = true; } } } if (!isPartitionKey) { String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType().toString()); schema.addColumn(fieldName, dataType); } } // validate field schema. HiveCatalogUtil.validateSchema(table); stats = new TableStats(); options = new KeyValueSet(); options.putAll(table.getParameters()); options.remove("EXTERNAL"); Properties properties = table.getMetadata(); if (properties != null) { // set field delimiter String fieldDelimiter = "", nullFormat = ""; if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) { fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM); } else { // if hive table used default row format delimiter, Properties doesn't have it. // So, Tajo must set as follows: fieldDelimiter = "\u0001"; } // set null format if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) { nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT); } else { nullFormat = "\\N"; } options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT); // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); storeType = CatalogUtil.getStoreType(HiveCatalogUtil.getStoreType(fileOutputformat)); if (storeType.equals(CatalogProtos.StoreType.TEXTFILE)) { options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) { options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (ColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE)) { options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinarySerDe.class.getName().equals(serde)) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (LazySimpleSerDe.class.getName().equals(serde)) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } // set data size long totalSize = 0; if (properties.getProperty("totalSize") != null) { totalSize = Long.parseLong(properties.getProperty("totalSize")); } else { try { FileSystem fs = path.getFileSystem(conf); if (fs.exists(path)) { totalSize = fs.getContentSummary(path).getLength(); } } catch (IOException ioe) { throw new CatalogException("Fail to get path. - path:" + path.toString(), ioe); } } stats.setNumBytes(totalSize); } // set partition keys List<FieldSchema> partitionKeys = table.getPartitionKeys(); if (null != partitionKeys) { org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); StringBuilder sb = new StringBuilder(); if (partitionKeys.size() > 0) { for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HiveCatalogUtil .getTajoFieldType(fieldSchema.getType().toString()); String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.addColumn(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); } sb.append(fieldSchema.getName()); } partitions = new PartitionMethodDesc(databaseName, tableName, PartitionType.COLUMN, sb.toString(), expressionSchema); } } } finally { if (client != null) client.release(); } TableMeta meta = new TableMeta(storeType, options); TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { tableDesc.setExternal(true); } if (stats != null) { tableDesc.setStats(stats); } if (partitions != null) { tableDesc.setPartitionMethod(partitions); } return tableDesc.getProto(); }
From source file:org.apache.tajo.engine.planner.LogicalPlanner.java
License:Apache License
private void updatePhysicalInfo(TableDesc desc) { if (desc.getPath() != null) { try {/*www. j a va 2s.c om*/ FileSystem fs = desc.getPath().getFileSystem(new Configuration()); FileStatus status = fs.getFileStatus(desc.getPath()); if (desc.getStats() != null && (status.isDirectory() || status.isFile())) { ContentSummary summary = fs.getContentSummary(desc.getPath()); if (summary != null) { long volume = summary.getLength(); desc.getStats().setNumBytes(volume); } } } catch (Throwable t) { LOG.warn(t); } } }
From source file:org.apache.tajo.engine.planner.rewrite.PartitionedTableRewriter.java
License:Apache License
private void updateTableStat(PartitionedTableScanNode scanNode) throws PlanningException { if (scanNode.getInputPaths().length > 0) { try {/*w w w .ja v a 2s .c o m*/ FileSystem fs = scanNode.getInputPaths()[0].getFileSystem(systemConf); long totalVolume = 0; for (Path input : scanNode.getInputPaths()) { ContentSummary summary = fs.getContentSummary(input); totalVolume += summary.getLength(); totalVolume += summary.getFileCount(); } scanNode.getTableDesc().getStats().setNumBytes(totalVolume); } catch (IOException e) { throw new PlanningException(e); } } }
From source file:org.apache.tajo.engine.query.TestTablePartitions.java
License:Apache License
@Test public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testInsertIntoColumnPartitionedTableByThreeColumns"); ResultSet res = testBase.execute("create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) "); res.close();//from w w w . ja va2 s . c om TajoTestingCluster cluster = testBase.getTestingCluster(); CatalogService catalog = cluster.getMaster().getCatalog(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); res = executeString("insert into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); Path path = new Path(desc.getPath()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } res = executeString("select * from " + tableName + " where col2 = 2"); Map<Double, int[]> resultRows1 = Maps.newHashMap(); resultRows1.put(45.0d, new int[] { 3, 2 }); resultRows1.put(38.0d, new int[] { 2, 2 }); for (int i = 0; i < 2; i++) { assertTrue(res.next()); assertEquals(resultRows1.get(res.getDouble(4))[0], res.getInt(2)); assertEquals(resultRows1.get(res.getDouble(4))[1], res.getInt(3)); } res.close(); Map<Double, int[]> resultRows2 = Maps.newHashMap(); resultRows2.put(49.0d, new int[] { 3, 3 }); resultRows2.put(45.0d, new int[] { 3, 2 }); resultRows2.put(38.0d, new int[] { 2, 2 }); res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2"); for (int i = 0; i < 3; i++) { assertTrue(res.next()); assertEquals(resultRows2.get(res.getDouble(4))[0], res.getInt(2)); assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3)); } res.close(); // insert into already exists partitioned table res = executeString("insert into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); path = new Path(desc.getPath()); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } String expected = "N\n" + "N\n" + "N\n" + "N\n" + "N\n" + "N\n" + "R\n" + "R\n" + "R\n" + "R\n"; String tableData = getTableFileContents(new Path(desc.getPath())); assertEquals(expected, tableData); res = executeString("select * from " + tableName + " where col2 = 2"); String resultSetData = resultSetToString(res); res.close(); expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,2,2,38.0\n" + "N,2,2,38.0\n" + "R,3,2,45.0\n" + "R,3,2,45.0\n"; assertEquals(expected, resultSetData); res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2"); resultSetData = resultSetToString(res); res.close(); expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,2,2,38.0\n" + "N,2,2,38.0\n" + "R,3,2,45.0\n" + "R,3,2,45.0\n" + "R,3,3,49.0\n" + "R,3,3,49.0\n"; assertEquals(expected, resultSetData); // Check not to remove existing partition directories. res = executeString("insert overwrite into " + tableName + " select l_returnflag, l_orderkey, l_partkey, 30.0 as l_quantity from lineitem " + " where l_orderkey = 1 and l_partkey = 1 and l_linenumber = 1"); res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=30.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); if (!testingCluster.isHCatalogStoreRunning()) { // TODO: If there is existing another partition directory, we must add its rows number to result row numbers. // assertEquals(6, desc.getStats().getNumRows().intValue()); } res = executeString("select * from " + tableName + " where col2 = 1"); resultSetData = resultSetToString(res); res.close(); expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,1,1,17.0\n" + "N,1,1,17.0\n" + "N,1,1,30.0\n" + "N,1,1,36.0\n" + "N,1,1,36.0\n"; assertEquals(expected, resultSetData); // insert overwrite empty result to partitioned table res = executeString("insert overwrite into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem where l_orderkey" + " > 100"); res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); ContentSummary summary = fs.getContentSummary(new Path(desc.getPath())); assertEquals(summary.getDirectoryCount(), 1L); assertEquals(summary.getFileCount(), 0L); assertEquals(summary.getLength(), 0L); }
From source file:org.apache.tajo.engine.query.TestUnionQuery.java
License:Apache License
private void verifyResultStats(Optional<TajoResultSetBase[]> existing, long numRows) throws Exception { assertTrue(existing.isPresent());/*from w w w . j a v a2 s . c o m*/ // Get TableStats using TajoResultSetBase. TajoResultSetBase[] resultSet = existing.get(); QueryId qid = resultSet[0].getQueryId(); QueryInfo queryInfo = testingCluster.getMaster().getContext().getQueryJobManager().getFinishedQuery(qid); TableDesc desc = queryInfo.getResultDesc(); TableStats stats = desc.getStats(); // Compare specified number of rows to the number of rows in TableStats. assertEquals(numRows, stats.getNumRows().longValue()); // Compare the volume number of directRaw to the number of rows in TableStats. FileSystem fs = FileSystem.get(conf); Path path = new Path(desc.getUri()); assertTrue(fs.exists(path)); ContentSummary summary = fs.getContentSummary(path); assertEquals(summary.getLength(), stats.getNumBytes().longValue()); closeResultSets(resultSet); }
From source file:org.apache.tajo.master.querymaster.Query.java
License:Apache License
public static long getTableVolume(TajoConf systemConf, Path tablePath) throws IOException { FileSystem fs = tablePath.getFileSystem(systemConf); ContentSummary directorySummary = fs.getContentSummary(tablePath); return directorySummary.getLength(); }
From source file:org.apache.tajo.plan.LogicalPlanner.java
License:Apache License
private void updatePhysicalInfo(TableDesc desc) { if (desc.getPath() != null && desc.getMeta().getStoreType() != StoreType.SYSTEM) { try {// w ww .j a v a 2s.c o m Path path = new Path(desc.getPath()); FileSystem fs = path.getFileSystem(new Configuration()); FileStatus status = fs.getFileStatus(path); if (desc.getStats() != null && (status.isDirectory() || status.isFile())) { ContentSummary summary = fs.getContentSummary(path); if (summary != null) { long volume = summary.getLength(); desc.getStats().setNumBytes(volume); } } } catch (Throwable t) { LOG.warn(t, t); } } }
From source file:org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter.java
License:Apache License
private void updateTableStat(OverridableConf queryContext, PartitionedTableScanNode scanNode) throws PlanningException { if (scanNode.getInputPaths().length > 0) { try {/*from w ww . j a v a 2s. c o m*/ FileSystem fs = scanNode.getInputPaths()[0].getFileSystem(queryContext.getConf()); long totalVolume = 0; for (Path input : scanNode.getInputPaths()) { ContentSummary summary = fs.getContentSummary(input); totalVolume += summary.getLength(); totalVolume += summary.getFileCount(); } scanNode.getTableDesc().getStats().setNumBytes(totalVolume); } catch (IOException e) { throw new PlanningException(e); } } }