List of usage examples for org.apache.hadoop.hdfs DistributedFileSystem delete
@Override
public boolean delete(Path f, final boolean recursive) throws IOException
From source file:com.cloudera.impala.service.Frontend.java
License:Apache License
/** * Loads a table or partition with one or more data files. If the "overwrite" flag * in the request is true, all existing data in the table/partition will be replaced. * If the "overwrite" flag is false, the files will be added alongside any existing * data files.// ww w . j a va 2 s. com */ public TLoadDataResp loadTableData(TLoadDataReq request) throws ImpalaException, IOException { TableName tableName = TableName.fromThrift(request.getTable_name()); // Get the destination for the load. If the load is targeting a partition, // this the partition location. Otherwise this is the table location. String destPathString = null; if (request.isSetPartition_spec()) { destPathString = impaladCatalog_ .getHdfsPartition(tableName.getDb(), tableName.getTbl(), request.getPartition_spec()) .getLocation(); } else { destPathString = impaladCatalog_.getTable(tableName.getDb(), tableName.getTbl()).getMetaStoreTable() .getSd().getLocation(); } Path destPath = new Path(destPathString); DistributedFileSystem dfs = FileSystemUtil.getDistributedFileSystem(destPath); // Create a temporary directory within the final destination directory to stage the // file move. Path tmpDestPath = FileSystemUtil.makeTmpSubdirectory(destPath); Path sourcePath = new Path(request.source_path); int filesLoaded = 0; if (dfs.isDirectory(sourcePath)) { filesLoaded = FileSystemUtil.moveAllVisibleFiles(sourcePath, tmpDestPath); } else { FileSystemUtil.moveFile(sourcePath, tmpDestPath, true); filesLoaded = 1; } // If this is an OVERWRITE, delete all files in the destination. if (request.isOverwrite()) { FileSystemUtil.deleteAllVisibleFiles(destPath); } // Move the files from the temporary location to the final destination. FileSystemUtil.moveAllVisibleFiles(tmpDestPath, destPath); // Cleanup the tmp directory. dfs.delete(tmpDestPath, true); TLoadDataResp response = new TLoadDataResp(); TColumnValue col = new TColumnValue(); String loadMsg = String.format("Loaded %d file(s). Total files in destination location: %d", filesLoaded, FileSystemUtil.getTotalNumVisibleFiles(destPath)); col.setString_val(loadMsg); response.setLoad_summary(new TResultRow(Lists.newArrayList(col))); return response; }
From source file:com.trace.hadoop.TestDFSRename.java
License:Apache License
/** * Perform operations such as setting quota, deletion of files, rename and * ensure system can apply edits log during startup. *//*from w ww . j ava 2s . co m*/ public void testEditsLog() throws Exception { DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); Path src1 = new Path(dir, "testEditsLog/srcdir/src1"); Path dst1 = new Path(dir, "testEditsLog/dstdir/dst1"); createFile(fs, src1); fs.mkdirs(dst1.getParent()); createFile(fs, dst1); // Set quota so that dst1 parent cannot allow under it new files/directories fs.setQuota(dst1.getParent(), 2, FSConstants.QUOTA_DONT_SET); // Free up quota for a subsequent rename fs.delete(dst1, true); rename(src1, dst1, true, false); // Restart the cluster and ensure the above operations can be // loaded from the edits log restartCluster(); fs = (DistributedFileSystem) cluster.getFileSystem(); assertFalse(fs.exists(src1)); // ensure src1 is already renamed assertTrue(fs.exists(dst1)); // ensure rename dst exists }
From source file:org.apache.tez.client.TestTezClientUtils.java
License:Apache License
public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, Configuration conf) throws Exception { Path topLevelDir = null;/*from www . j a v a 2s .co m*/ try { FsPermission publicDirPerms = new FsPermission((short) 0755); // rwxr-xr-x FsPermission privateDirPerms = new FsPermission((short) 0754); // rwxr-xr-- FsPermission publicFilePerms = new FsPermission((short) 0554); // r-xr-xr-- FsPermission privateFilePerms = new FsPermission((short) 0550); // r-xr-x--- String fsURI = remoteFs.getUri().toString(); topLevelDir = new Path(fsURI, "/testLRVisibility"); Assert.assertTrue(remoteFs.mkdirs(topLevelDir, publicDirPerms)); Path publicSubDir = new Path(topLevelDir, "public_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(publicSubDir, publicDirPerms)); Path privateSubDir = new Path(topLevelDir, "private_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(privateSubDir, privateDirPerms)); Path publicFile = new Path(publicSubDir, "public_file"); Assert.assertTrue(remoteFs.createNewFile(publicFile)); remoteFs.setPermission(publicFile, publicFilePerms); Path privateFile = new Path(publicSubDir, "private_file"); Assert.assertTrue(remoteFs.createNewFile(privateFile)); remoteFs.setPermission(privateFile, privateFilePerms); Path publicFileInPrivateSubdir = new Path(privateSubDir, "public_file_in_private_subdir"); Assert.assertTrue(remoteFs.createNewFile(publicFileInPrivateSubdir)); remoteFs.setPermission(publicFileInPrivateSubdir, publicFilePerms); TezConfiguration tezConf = new TezConfiguration(conf); String tmpTezLibUris = String.format("%s,%s,%s,%s", topLevelDir, publicSubDir, privateSubDir, conf.get(TezConfiguration.TEZ_LIB_URIS, "")); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tmpTezLibUris); Map<String, LocalResource> lrMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(publicFile.getName(), LocalResourceVisibility.PUBLIC, lrMap.get(publicFile.getName()).getVisibility()); Assert.assertEquals(privateFile.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(privateFile.getName()).getVisibility()); Assert.assertEquals(publicFileInPrivateSubdir.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(publicFileInPrivateSubdir.getName()).getVisibility()); // test tar.gz tezConf = new TezConfiguration(conf); Path tarFile = new Path(topLevelDir, "foo.tar.gz"); Assert.assertTrue(remoteFs.createNewFile(tarFile)); //public remoteFs.setPermission(tarFile, publicFilePerms); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tarFile.toString()); lrMap.clear(); Assert.assertTrue(TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap)); Assert.assertEquals(LocalResourceVisibility.PUBLIC, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); //private remoteFs.setPermission(tarFile, privateFilePerms); lrMap.clear(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(LocalResourceVisibility.PRIVATE, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); } finally { if (topLevelDir != null) { remoteFs.delete(topLevelDir, true); } } }
From source file:se.sics.gvod.stream.system.hops.SetupExperiment.java
License:Open Source License
public static void main(String[] args) throws IOException, HashUtil.HashBuilderException { String hopsURL = "bbc1.sics.se:26801"; Configuration conf = new Configuration(); conf.set("fs.defaultFS", hopsURL); DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(conf); String path = "/experiment"; if (!fs.isDirectory(new Path(path))) { fs.mkdirs(new Path(path)); } else {//from w ww . j a v a 2 s.co m fs.delete(new Path(path), true); fs.mkdirs(new Path(path)); } String uploadDirPath = path + "/upload"; fs.mkdirs(new Path(uploadDirPath)); String downloadDirPath = path + "/download"; fs.mkdirs(new Path(downloadDirPath)); String dataFile = uploadDirPath + "/file"; Random rand = new Random(1234); try (FSDataOutputStream out = fs.create(new Path(dataFile))) { for (int i = 0; i < fileSize / pieceSize; i++) { byte[] data = new byte[1024]; rand.nextBytes(data); out.write(data); out.flush(); } System.err.println("created file - expected:" + fileSize + " created:" + out.size()); } catch (IOException ex) { throw new RuntimeException(ex); } fs.close(); }