List of usage examples for org.apache.hadoop.fs FileContext delete
public boolean delete(final Path f, final boolean recursive) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException
From source file:com.datatorrent.stram.FSRecoveryHandler.java
License:Apache License
@Override public Object restore() throws IOException { FileContext fc = FileContext.getFileContext(fs.getUri()); // recover from wherever it was left if (fc.util().exists(snapshotBackupPath)) { LOG.warn("Incomplete checkpoint, reverting to {}", snapshotBackupPath); fc.rename(snapshotBackupPath, snapshotPath, Rename.OVERWRITE); // combine logs (w/o append, create new file) Path tmpLogPath = new Path(basedir, "log.combined"); FSDataOutputStream fsOut = fc.create(tmpLogPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); try {/*from w w w.ja va 2 s. c om*/ FSDataInputStream fsIn = fc.open(logBackupPath); try { IOUtils.copy(fsIn, fsOut); } finally { fsIn.close(); } fsIn = fc.open(logPath); try { IOUtils.copy(fsIn, fsOut); } finally { fsIn.close(); } } finally { fsOut.close(); } fc.rename(tmpLogPath, logPath, Rename.OVERWRITE); fc.delete(logBackupPath, false); } else { // we have log backup, but no checkpoint backup // failure between log rotation and writing checkpoint if (fc.util().exists(logBackupPath)) { LOG.warn("Found {}, did checkpointing fail?", logBackupPath); fc.rename(logBackupPath, logPath, Rename.OVERWRITE); } } if (!fc.util().exists(snapshotPath)) { LOG.debug("No existing checkpoint."); return null; } LOG.debug("Reading checkpoint {}", snapshotPath); InputStream is = fc.open(snapshotPath); // indeterministic class loading behavior // http://stackoverflow.com/questions/9110677/readresolve-not-working-an-instance-of-guavas-serializedform-appears final ClassLoader loader = Thread.currentThread().getContextClassLoader(); ObjectInputStream ois = new ObjectInputStream(is) { @Override protected Class<?> resolveClass(ObjectStreamClass objectStreamClass) throws IOException, ClassNotFoundException { return Class.forName(objectStreamClass.getName(), true, loader); } }; //ObjectInputStream ois = new ObjectInputStream(is); try { return ois.readObject(); } catch (ClassNotFoundException cnfe) { throw new IOException("Failed to read checkpointed state", cnfe); } finally { ois.close(); } }
From source file:com.ikanow.aleph2.management_db.services.DataBucketCrudService.java
License:Apache License
/** Deletes the entire bucket, ie data and state * @param to_delete/* ww w . j a v a 2 s. c o m*/ * @param storage_service */ public static void removeBucketPath(final DataBucketBean to_delete, final IStorageService storage_service, Optional<String> extra_path) throws Exception { final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()) .get(); final String bucket_root = storage_service.getBucketRootPath() + "/" + to_delete.full_name() + IStorageService.BUCKET_SUFFIX; dfs.delete(new Path(bucket_root + extra_path.map(s -> "/" + s).orElse("")), true); }
From source file:com.ikanow.aleph2.management_db.services.SharedLibraryCrudService.java
License:Apache License
@Override public ManagementFuture<Boolean> deleteObjectBySpec(QueryComponent<SharedLibraryBean> unique_spec) { return FutureUtils.createManagementFuture( _underlying_library_db.get().getObjectBySpec(unique_spec).thenCompose(lib -> { if (lib.isPresent()) { try { final FileContext fs = _storage_service.get() .getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get(); fs.delete(fs.makeQualified(new Path(lib.get().path_name())), false); } catch (Exception e) { // i suppose we don't really care if it fails.. // (maybe add a message?) //DEBUG //e.printStackTrace(); }//w w w. j a v a2 s.c om return _underlying_library_db.get().deleteObjectBySpec(unique_spec); } else { return CompletableFuture.completedFuture(false); } })); }
From source file:com.ikanow.aleph2.storage_service_hdfs.services.TestMockHdfsStorageSystem.java
License:Apache License
@Test public void test_basic_secondaryBuffers() throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IllegalArgumentException, IOException { // 0) Setup//from w w w.j ava 2 s. c om final String temp_dir = System.getProperty("java.io.tmpdir") + File.separator; final GlobalPropertiesBean globals = BeanTemplateUtils.build(GlobalPropertiesBean.class) .with(GlobalPropertiesBean::local_yarn_config_dir, temp_dir) .with(GlobalPropertiesBean::distributed_root_dir, temp_dir) .with(GlobalPropertiesBean::local_root_dir, temp_dir) .with(GlobalPropertiesBean::distributed_root_dir, temp_dir).done().get(); final MockHdfsStorageService storage_service = new MockHdfsStorageService(globals); // Some buckets final DataBucketBean bucket = BeanTemplateUtils.build(DataBucketBean.class) .with(DataBucketBean::full_name, "/test/storage/bucket") .with(DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class).done().get()) .done().get(); setup_bucket(storage_service, bucket, Collections.emptyList()); // Get primary buffer doesn't work: assertFalse(storage_service.getDataService().get().getPrimaryBufferName(bucket).isPresent()); // Add some secondary buffers and check they get picked up final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()) .get(); final String bucket_root = storage_service.getBucketRootPath() + "/" + bucket.full_name(); dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "test1"), FsPermission.getDirDefault(), true); //(skip the current dir once just to check it doesn't cause problems) dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test2"), FsPermission.getDirDefault(), true); dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON), FsPermission.getDirDefault(), true); dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "test3"), FsPermission.getDirDefault(), true); dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED), FsPermission.getDirDefault(), true); assertEquals(Arrays.asList("test1", "test2", "test3"), storage_service.getDataService().get() .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList())); //(check dedups) dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test1"), FsPermission.getDirDefault(), true); assertEquals(Arrays.asList("test1", "test2", "test3"), storage_service.getDataService().get() .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList())); try { dfs.delete(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "test3"), true); } catch (Exception e) { } assertEquals(Arrays.asList("test1", "test2"), storage_service.getDataService().get() .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList())); try { dfs.delete(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "test1"), true); } catch (Exception e) { } assertEquals(Arrays.asList("test1", "test2"), storage_service.getDataService().get() .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList())); }
From source file:com.ikanow.aleph2.storage_service_hdfs.services.TestMockHdfsStorageSystem.java
License:Apache License
/** * @param storage_service/*from w w w. j ava 2s . c om*/ * @param bucket * @param extra_suffixes - start with $ to indicate a new secondary buffer, else is a normal suffix */ protected void setup_bucket(MockHdfsStorageService storage_service, final DataBucketBean bucket, List<String> extra_suffixes) { final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()) .get(); final String bucket_root = storage_service.getBucketRootPath() + "/" + bucket.full_name(); //(first delete root path) try { dfs.delete(new Path(bucket_root), true); } catch (Exception e) { } Stream.concat( Arrays.asList("/managed_bucket", "/managed_bucket/logs", "/managed_bucket/logs/harvest", "/managed_bucket/logs/enrichment", "/managed_bucket/logs/storage", "/managed_bucket/assets", "/managed_bucket/import", "/managed_bucket/import/stored", "/managed_bucket/import/stored/raw/current", "/managed_bucket/import/stored/json/current", "/managed_bucket/import/stored/processed/current", "/managed_bucket/import/transient/current", "/managed_bucket/import/ready", "/managed_bucket/import/temp").stream(), extra_suffixes.stream() .flatMap( s -> s.startsWith("$") ? Stream.of("/managed_bucket/import/stored/raw/" + s.substring(1), "/managed_bucket/import/stored/json/" + s.substring(1), "/managed_bucket/import/stored/processed/" + s.substring(1), "/managed_bucket/import/transient/" + s.substring(1)) : Stream.of(s))) .map(s -> new Path(bucket_root + s)) .forEach(Lambdas.wrap_consumer_u(p -> dfs.mkdir(p, FsPermission.getDefault(), true))); }
From source file:io.hops.tensorflow.TestCluster.java
License:Apache License
protected void setupInternal(int numNodeManager) throws Exception { LOG.info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.set("yarn.log-aggregation-enable", "true"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1); yarnCluster.init(conf);// w w w. j a va 2 s.c o m yarnCluster.start(); conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort()); waitForNMsToRegister(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(url.getPath())); os.write(bytesOut.toByteArray()); os.close(); } FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }
From source file:io.hops.tensorflow.TestCluster.java
License:Apache License
@After public void tearDown() throws IOException { if (yarnCluster != null) { try {/*from www.j a v a 2 s. co m*/ yarnCluster.stop(); } finally { yarnCluster = null; } } FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); }
From source file:org.apache.metron.integration.components.YarnComponent.java
License:Apache License
@Override public void start() throws UnableToStartException { conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); try {//from w ww . ja va2 s . c om yarnCluster = new MiniYARNCluster(testName, 1, NUM_NMS, 1, 1, true); yarnCluster.init(conf); yarnCluster.start(); waitForNMsToRegister(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(url.getPath())); os.write(bytesOut.toByteArray()); os.close(); FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); try { Thread.sleep(2000); } catch (InterruptedException e) { } } catch (Exception e) { throw new UnableToStartException("Exception setting up yarn cluster", e); } }
From source file:org.apache.metron.integration.components.YarnComponent.java
License:Apache License
@Override public void stop() { if (yarnCluster != null) { try {/*from w w w. j ava 2 s .co m*/ yarnCluster.stop(); } finally { yarnCluster = null; } } try { FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); } catch (Exception e) { } }
From source file:org.apache.solr.hadoop.hack.MiniMRYarnCluster.java
License:Apache License
@Override public void serviceInit(Configuration conf) throws Exception { conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath()); }/*from ww w. j a v a2s .c om*/ // By default, VMEM monitoring disabled, PMEM monitoring enabled. if (!conf.getBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) { conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); } conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); try { Path stagingPath = FileContext.getFileContext(conf) .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR))); /* * Re-configure the staging path on Windows if the file system is localFs. * We need to use a absolute path that contains the drive letter. The unit * test could run on a different drive than the AM. We can run into the * issue that job files are localized to the drive where the test runs on, * while the AM starts on a different drive and fails to find the job * metafiles. Using absolute path can avoid this ambiguity. */ if (Path.WINDOWS) { if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath()); } } FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); } LOG.info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.mkdir(stagingPath, null, true); //mkdir done directory as well String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.makeQualified(new Path(doneDir)); fc.mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID }); conf.setClass(String.format(Locale.ENGLISH, YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class); // Non-standard shuffle port conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); super.serviceInit(conf); }