List of usage examples for org.apache.hadoop.fs ContentSummary getDirectoryCount
public long getDirectoryCount()
From source file:com.bigstep.datalake.JsonUtil.java
License:Apache License
/** Convert a ContentSummary to a Json string. */ public static String toJsonString(final ContentSummary contentsummary) { if (contentsummary == null) { return null; }//from w w w .j av a 2 s . c o m final Map<String, Object> m = new TreeMap<String, Object>(); m.put("length", contentsummary.getLength()); m.put("fileCount", contentsummary.getFileCount()); m.put("directoryCount", contentsummary.getDirectoryCount()); m.put("quota", contentsummary.getQuota()); m.put("spaceConsumed", contentsummary.getSpaceConsumed()); m.put("spaceQuota", contentsummary.getSpaceQuota()); return toJsonString(ContentSummary.class, m); }
From source file:org.apache.ignite.igfs.IgfsHadoopFileSystemAbstractSelfTest.java
License:Apache License
/** * Compare content of two folders./* ww w. j a v a 2 s . c om*/ * * @param cfg Paths configuration to compare. * @throws IOException If failed. */ @SuppressWarnings("deprecation") private void compareContent(Config cfg) throws IOException { Deque<Config> queue = new LinkedList<>(); queue.add(cfg); for (Config c = queue.poll(); c != null; c = queue.poll()) { boolean exists; assertEquals("Check existence [src=" + c.src + ", dest=" + c.dest + ']', exists = c.srcFs.exists(c.src), c.destFs.exists(c.dest)); assertEquals("Check types (files?) [src=" + c.src + ", dest=" + c.dest + ']', c.srcFs.isFile(c.src), c.destFs.isFile(c.dest)); if (exists) { ContentSummary srcSummary = c.srcFs.getContentSummary(c.src); ContentSummary dstSummary = c.destFs.getContentSummary(c.dest); assertEquals("Directories number comparison failed", srcSummary.getDirectoryCount(), dstSummary.getDirectoryCount()); assertEquals("Files number comparison failed", srcSummary.getFileCount(), dstSummary.getFileCount()); assertEquals("Space consumed comparison failed", srcSummary.getSpaceConsumed(), dstSummary.getSpaceConsumed()); assertEquals("Length comparison failed", srcSummary.getLength(), dstSummary.getLength()); // Intentionally skipping quotas checks as they can vary. } else { assertContentSummaryFails(c.srcFs, c.src); assertContentSummaryFails(c.destFs, c.dest); } if (!exists) continue; FileStatus[] srcSt = c.srcFs.listStatus(c.src); FileStatus[] destSt = c.destFs.listStatus(c.dest); assert srcSt != null && destSt != null : "Both not null" + " [srcSt=" + Arrays.toString(srcSt) + ", destSt=" + Arrays.toString(destSt) + ']'; assertEquals("Check listing [src=" + c.src + ", dest=" + c.dest + ']', srcSt.length, destSt.length); // Listing of the file returns the only element with this file. if (srcSt.length == 1 && c.src.equals(srcSt[0].getPath())) { assertEquals(c.dest, destSt[0].getPath()); assertTrue("Expects file [src=" + c.src + ", srcSt[0]=" + srcSt[0] + ']', !srcSt[0].isDir()); assertTrue("Expects file [dest=" + c.dest + ", destSt[0]=" + destSt[0] + ']', !destSt[0].isDir()); FSDataInputStream srcIn = null; FSDataInputStream destIn = null; try { srcIn = c.srcFs.open(c.src); destIn = c.destFs.open(c.dest); GridTestIoUtils.assertEqualStreams(srcIn, destIn, srcSt[0].getLen()); } finally { U.closeQuiet(srcIn); U.closeQuiet(destIn); } continue; // Skip the following directories validations. } // Sort both arrays. Arrays.sort(srcSt, STATUS_COMPARATOR); Arrays.sort(destSt, STATUS_COMPARATOR); for (int i = 0; i < srcSt.length; i++) // Dig in deep to the last leaf, instead of collecting full tree in memory. queue.addFirst(new Config(c.srcFs, srcSt[i].getPath(), c.destFs, destSt[i].getPath())); // Add non-existent file to check in the current folder. String rndFile = "Non-existent file #" + UUID.randomUUID().toString(); queue.addFirst(new Config(c.srcFs, new Path(c.src, rndFile), c.destFs, new Path(c.dest, rndFile))); } }
From source file:org.apache.tajo.engine.query.TestTablePartitions.java
License:Apache License
@Test public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testInsertIntoColumnPartitionedTableByThreeColumns"); ResultSet res = testBase.execute("create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) "); res.close();/*from w w w.j a va 2 s . c om*/ TajoTestingCluster cluster = testBase.getTestingCluster(); CatalogService catalog = cluster.getMaster().getCatalog(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); res = executeString("insert into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); Path path = new Path(desc.getPath()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } res = executeString("select * from " + tableName + " where col2 = 2"); Map<Double, int[]> resultRows1 = Maps.newHashMap(); resultRows1.put(45.0d, new int[] { 3, 2 }); resultRows1.put(38.0d, new int[] { 2, 2 }); for (int i = 0; i < 2; i++) { assertTrue(res.next()); assertEquals(resultRows1.get(res.getDouble(4))[0], res.getInt(2)); assertEquals(resultRows1.get(res.getDouble(4))[1], res.getInt(3)); } res.close(); Map<Double, int[]> resultRows2 = Maps.newHashMap(); resultRows2.put(49.0d, new int[] { 3, 3 }); resultRows2.put(45.0d, new int[] { 3, 2 }); resultRows2.put(38.0d, new int[] { 2, 2 }); res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2"); for (int i = 0; i < 3; i++) { assertTrue(res.next()); assertEquals(resultRows2.get(res.getDouble(4))[0], res.getInt(2)); assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3)); } res.close(); // insert into already exists partitioned table res = executeString("insert into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); path = new Path(desc.getPath()); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } String expected = "N\n" + "N\n" + "N\n" + "N\n" + "N\n" + "N\n" + "R\n" + "R\n" + "R\n" + "R\n"; String tableData = getTableFileContents(new Path(desc.getPath())); assertEquals(expected, tableData); res = executeString("select * from " + tableName + " where col2 = 2"); String resultSetData = resultSetToString(res); res.close(); expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,2,2,38.0\n" + "N,2,2,38.0\n" + "R,3,2,45.0\n" + "R,3,2,45.0\n"; assertEquals(expected, resultSetData); res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2"); resultSetData = resultSetToString(res); res.close(); expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,2,2,38.0\n" + "N,2,2,38.0\n" + "R,3,2,45.0\n" + "R,3,2,45.0\n" + "R,3,3,49.0\n" + "R,3,3,49.0\n"; assertEquals(expected, resultSetData); // Check not to remove existing partition directories. res = executeString("insert overwrite into " + tableName + " select l_returnflag, l_orderkey, l_partkey, 30.0 as l_quantity from lineitem " + " where l_orderkey = 1 and l_partkey = 1 and l_linenumber = 1"); res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); assertTrue(fs.isDirectory(path)); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=30.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); if (!testingCluster.isHCatalogStoreRunning()) { // TODO: If there is existing another partition directory, we must add its rows number to result row numbers. // assertEquals(6, desc.getStats().getNumRows().intValue()); } res = executeString("select * from " + tableName + " where col2 = 1"); resultSetData = resultSetToString(res); res.close(); expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,1,1,17.0\n" + "N,1,1,17.0\n" + "N,1,1,30.0\n" + "N,1,1,36.0\n" + "N,1,1,36.0\n"; assertEquals(expected, resultSetData); // insert overwrite empty result to partitioned table res = executeString("insert overwrite into " + tableName + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem where l_orderkey" + " > 100"); res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); ContentSummary summary = fs.getContentSummary(new Path(desc.getPath())); assertEquals(summary.getDirectoryCount(), 1L); assertEquals(summary.getFileCount(), 0L); assertEquals(summary.getLength(), 0L); }
From source file:org.exem.flamingo.agent.nn.hdfs.HdfsFileInfo.java
License:Apache License
public HdfsFileInfo(FileStatus fileStatus, ContentSummary contentSummary) { this.fullyQualifiedPath = fileStatus.getPath().toUri().getPath(); this.filename = isEmpty(getFilename(fullyQualifiedPath)) ? getDirectoryName(fullyQualifiedPath) : getFilename(fullyQualifiedPath); this.length = fileStatus.isFile() ? fileStatus.getLen() : contentSummary.getLength(); this.path = getPath(fullyQualifiedPath); this.directory = fileStatus.isDirectory(); this.file = !fileStatus.isDirectory(); this.owner = fileStatus.getOwner(); this.group = fileStatus.getGroup(); this.blockSize = fileStatus.getBlockSize(); this.replication = fileStatus.getReplication(); this.modificationTime = fileStatus.getModificationTime(); if (contentSummary != null) { this.spaceConsumed = contentSummary.getSpaceConsumed(); this.spaceQuota = contentSummary.getSpaceQuota(); this.quota = contentSummary.getQuota(); this.directoryCount = contentSummary.getDirectoryCount(); this.fileCount = contentSummary.getFileCount(); }//from w w w.j a v a2 s.c o m this.accessTime = fileStatus.getAccessTime(); this.permission = fileStatus.getPermission().toString(); }
From source file:org.opencloudengine.garuda.model.HdfsFileInfo.java
License:Open Source License
public HdfsFileInfo(FileStatus fileStatus, ContentSummary contentSummary) { this.fullyQualifiedPath = fileStatus.getPath().toUri().getPath(); this.filename = isEmpty(getFilename(fullyQualifiedPath)) ? getDirectoryName(fullyQualifiedPath) : getFilename(fullyQualifiedPath); this.length = fileStatus.getLen(); this.path = getPath(fullyQualifiedPath); this.directory = fileStatus.isDirectory(); this.file = !fileStatus.isDirectory(); this.owner = fileStatus.getOwner(); this.group = fileStatus.getGroup(); this.blockSize = fileStatus.getBlockSize(); this.replication = fileStatus.getReplication(); this.modificationTime = fileStatus.getModificationTime(); if (contentSummary != null) { this.spaceConsumed = contentSummary.getSpaceConsumed(); this.quota = contentSummary.getQuota(); this.spaceQuota = contentSummary.getSpaceQuota(); this.directoryCount = contentSummary.getDirectoryCount(); this.fileCount = contentSummary.getFileCount(); }/*www.j a v a 2s . co m*/ this.accessTime = fileStatus.getAccessTime(); this.permission = fileStatus.getPermission().toString(); }
From source file:org.openflamingo.fs.hdfs.HdfsFileSystemProvider.java
License:Apache License
@Override public FileInfo getFileInfo(String path) { try {//from w w w .ja va 2 s . co m FileStatus fileStatus = fs.getFileStatus(new Path(path)); HdfsFileInfo hdfsFileInfo = new HdfsFileInfo(fileStatus); ContentSummary summary = fs.getContentSummary(new Path(path)); hdfsFileInfo.setBlockSize(fileStatus.getBlockSize()); hdfsFileInfo.setReplication(fileStatus.getReplication()); hdfsFileInfo.setDirectoryCount(summary.getDirectoryCount()); hdfsFileInfo.setFileCount(summary.getFileCount()); hdfsFileInfo.setQuota(summary.getQuota()); hdfsFileInfo.setSpaceQuota(summary.getSpaceQuota()); hdfsFileInfo.setSpaceConsumed(StringUtils.byteDesc(summary.getSpaceConsumed())); hdfsFileInfo.setLength(summary.getLength()); return hdfsFileInfo; } catch (Exception ex) { throw new FileSystemException(bundle.message("S_FS", "CANNOT_GET_FILE_INFO", path), ex); } }
From source file:org.openflamingo.remote.thrift.thriftfs.ThriftUtils.java
License:Apache License
public static ContentSummary toThrift(org.apache.hadoop.fs.ContentSummary cs, String path) { ContentSummary tcs = new ContentSummary(); tcs.fileCount = cs.getFileCount();/*from w ww . ja v a2s . c o m*/ tcs.directoryCount = cs.getDirectoryCount(); tcs.quota = cs.getQuota(); tcs.spaceConsumed = cs.getSpaceConsumed(); tcs.spaceQuota = cs.getSpaceQuota(); tcs.path = path; return tcs; }
From source file:org.pentaho.di.job.entries.hadooptransjobexecutor.DistributedCacheUtilTest.java
License:Apache License
/** * Utility to attempt to stage a file to HDFS for use with Distributed Cache. * * @param ch Distributed Cache Helper * @param source File or directory to stage * @param fs FileSystem to stage to * @param root Root directory to clean up when this test is complete * @param dest Destination path to stage to * @param expectedFileCount Expected number of files to exist in the destination once staged * @param expectedDirCount Expected number of directories to exist in the destiation once staged * @throws Exception/* www .ja v a 2 s .c o m*/ */ private void stageForCacheTester(DistributedCacheUtil ch, FileObject source, FileSystem fs, Path root, Path dest, int expectedFileCount, int expectedDirCount) throws Exception { try { ch.stageForCache(source, fs, dest, true); assertTrue(fs.exists(dest)); ContentSummary cs = fs.getContentSummary(dest); assertEquals(expectedFileCount, cs.getFileCount()); assertEquals(expectedDirCount, cs.getDirectoryCount()); assertEquals(FsPermission.createImmutable((short) 0755), fs.getFileStatus(dest).getPermission()); } finally { // Clean up after ourself if (!fs.delete(root, true)) { System.err.println("error deleting FileSystem temp dir " + root); } } }
From source file:org.pentaho.di.job.entries.hadooptransjobexecutor.DistributedCacheUtilTest.java
License:Apache License
@Test public void stagePluginsForCache() throws Exception { DistributedCacheUtil ch = new DistributedCacheUtil(); Configuration conf = new Configuration(); org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.getLocal(conf); Path pluginsDir = new Path("bin/test/plugins-installation-dir"); FileObject pluginDir = createTestFolderWithContent(); try {/*www. j a v a 2s . c o m*/ ch.stagePluginsForCache(fs, pluginsDir, true, Arrays.asList(pluginDir)); Path pluginInstallPath = new Path(pluginsDir, pluginDir.getURL().toURI().getPath()); assertTrue(fs.exists(pluginInstallPath)); ContentSummary summary = fs.getContentSummary(pluginInstallPath); assertEquals(3, summary.getFileCount()); assertEquals(2, summary.getDirectoryCount()); } finally { pluginDir.delete(new AllFileSelector()); fs.delete(pluginsDir, true); } }
From source file:org.pentaho.hadoop.shim.common.DistributedCacheTestUtil.java
License:Apache License
/** * Utility to attempt to stage a file to HDFS for use with Distributed Cache. * * @param ch Distributed Cache Helper * @param source File or directory to stage * @param fs FileSystem to stage to * @param root Root directory to clean up when this test is complete * @param dest Destination path to stage to * @param expectedFileCount Expected number of files to exist in the destination once staged * @param expectedDirCount Expected number of directories to exist in the destiation once staged * @throws Exception/*from w w w.ja v a 2 s. c o m*/ */ static void stageForCacheTester(DistributedCacheUtilImpl ch, FileObject source, FileSystem fs, Path root, Path dest, int expectedFileCount, int expectedDirCount) throws Exception { try { ch.stageForCache(source, fs, dest, true); assertTrue(fs.exists(dest)); ContentSummary cs = fs.getContentSummary(dest); assertEquals(expectedFileCount, cs.getFileCount()); assertEquals(expectedDirCount, cs.getDirectoryCount()); assertEquals(FsPermission.createImmutable((short) 0755), fs.getFileStatus(dest).getPermission()); } finally { // Clean up after ourself if (!fs.delete(root, true)) { System.err.println("error deleting FileSystem temp dir " + root); } } }