List of usage examples for org.apache.hadoop.fs FileContext getFileStatus
public FileStatus getFileStatus(final Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException
From source file:com.ikanow.aleph2.aleph2_rest_utils.DataStoreCrudService.java
License:Apache License
/** * Utility for checking if a file exists or not, attempts to get * the paths status and returns true if that is successful and * its either a file or dir.// www. j a v a 2 s .c o m * * If a filenotfound exception is thrown, returns false * * Lets all other exceptions raise * * This is used to get around FileContext.delete not reporting correctly if a file has been successfully deleted. * @param path * @param context * @return * @throws AccessControlException * @throws UnsupportedFileSystemException * @throws IOException */ private static boolean doesPathExist(final Path path, final FileContext context) throws AccessControlException, UnsupportedFileSystemException, IOException { try { FileStatus status = context.getFileStatus(path); return status.isFile() || status.isDirectory(); } catch (FileNotFoundException e) { return false; } }
From source file:com.ikanow.aleph2.analytics.hadoop.services.BeJobLauncher.java
License:Open Source License
/** Cache the system and user classpaths * @param job/*from w ww .ja va 2 s . c om*/ * @param context * @throws IOException * @throws ExecutionException * @throws InterruptedException * @throws IllegalArgumentException */ protected static void cacheJars(final Job job, final DataBucketBean bucket, final IAnalyticsContext context) throws IllegalArgumentException, InterruptedException, ExecutionException, IOException { final FileContext fc = context.getServiceContext().getStorageService() .getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get(); final String rootPath = context.getServiceContext().getStorageService().getRootPath(); // Aleph2 libraries: need to cache them context.getAnalyticsContextLibraries(Optional.empty()).stream().map(f -> new File(f)) .map(f -> Tuples._2T(f, new Path(rootPath + "/" + f.getName()))).map(Lambdas.wrap_u(f_p -> { final FileStatus fs = Lambdas.get(() -> { try { return fc.getFileStatus(f_p._2()); } catch (Exception e) { return null; } }); if (null == fs) { //cache doesn't exist // Local version Path srcPath = FileContext.getLocalFSFileContext() .makeQualified(new Path(f_p._1().toString())); fc.util().copy(srcPath, f_p._2()); } return f_p._2(); })).forEach(Lambdas.wrap_consumer_u(path -> job.addFileToClassPath(path))); ; // User libraries: this is slightly easier since one of the 2 keys // is the HDFS path (the other is the _id) context.getAnalyticsLibraries(Optional.of(bucket), bucket.analytic_thread().jobs()).get().entrySet() .stream().map(kv -> kv.getKey()).filter(path -> path.startsWith(rootPath)) .forEach(Lambdas.wrap_consumer_u(path -> job.addFileToClassPath(new Path(path)))); ; }
From source file:com.ikanow.aleph2.analytics.r.services.BeJobLauncher.java
License:Apache License
/** Cache the system and user classpaths * @param job/* w w w . j ava 2s.c om*/ * @param context * @throws IOException * @throws ExecutionException * @throws InterruptedException * @throws IllegalArgumentException */ protected static void cacheJars(final Job job, final DataBucketBean bucket, final IAnalyticsContext context) throws IllegalArgumentException, InterruptedException, ExecutionException, IOException { final FileContext fc = context.getServiceContext().getStorageService() .getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get(); final String rootPath = context.getServiceContext().getStorageService().getRootPath(); // Aleph2 libraries: need to cache them context.getAnalyticsContextLibraries(Optional.empty()).stream().map(f -> new File(f)) .map(f -> Tuples._2T(f, new Path(rootPath + "/" + f.getName()))).map(Lambdas.wrap_u(f_p -> { final FileStatus fs = Lambdas.get(() -> { //TODO (ALEPH-12): need to clear out the cache intermittently try { return fc.getFileStatus(f_p._2()); } catch (Exception e) { return null; } }); if (null == fs) { //cache doesn't exist // Local version try (FSDataOutputStream outer = fc.create(f_p._2(), EnumSet.of(CreateFlag.CREATE), // ie should fail if the destination file already exists org.apache.hadoop.fs.Options.CreateOpts.createParent())) { Files.copy(f_p._1(), outer.getWrappedStream()); } catch (FileAlreadyExistsException e) {//(carry on - the file is versioned so it can't be out of date) } } return f_p._2(); })).forEach(Lambdas.wrap_consumer_u(path -> job.addFileToClassPath(path))); ; // User libraries: this is slightly easier since one of the 2 keys // is the HDFS path (the other is the _id) context.getAnalyticsLibraries(Optional.of(bucket), bucket.analytic_thread().jobs()).get().entrySet() .stream().map(kv -> kv.getKey()).filter(path -> path.startsWith(rootPath)) .forEach(Lambdas.wrap_consumer_u(path -> job.addFileToClassPath(new Path(path)))); ; }
From source file:com.ikanow.aleph2.analytics.spark.utils.SparkTechnologyUtils.java
License:Apache License
/** Checks if an aleph2 JAR file is cached * @param local_file//from www. ja v a 2 s . c o m * @param root_path * @param fc * @return */ public static Tuple3<File, Path, FileStatus> checkCache(final String local_file, final String root_path, final FileContext fc) { final File f = new File(local_file); final Tuple2<File, Path> f_p = Tuples._2T(f, new Path(root_path + "/" + f.getName())); final FileStatus fs = Lambdas.get(() -> { try { return fc.getFileStatus(f_p._2()); } catch (Exception e) { return null; } }); return Tuples._3T(f_p._1(), f_p._2(), fs); }
From source file:com.ikanow.aleph2.analytics.spark.utils.SparkTechnologyUtils.java
License:Apache License
/**Checks if an aleph2 JAR file with bad internal JARs is cached, does some excising if not * @param local_file/*w w w. j a va2 s .c o m*/ * @param root_path * @param fc * @return * @throws IOException */ public static Tuple3<File, Path, FileStatus> removeSparkConflictsAndCache(final String local_file, final String root_path, final FileContext fc) throws IOException { final String renamed_local_file = local_file.replaceFirst(".jar", "_sparkVersion.jar"); final File f = new File(renamed_local_file); final Tuple2<File, Path> f_p = Tuples._2T(f, new Path(root_path + "/" + f.getName())); final FileStatus fs = Lambdas.get(() -> { try { return fc.getFileStatus(f_p._2()); } catch (Exception e) { return null; } }); if (null == fs) { //cache doesn't exist // Create a new copy without akka in tmp File f_tmp = File.createTempFile("aleph2-temp", ".jar"); JarBuilderUtil.mergeJars(Arrays.asList(local_file), f_tmp.toString(), ImmutableSet.of("akka", "scala", "com/typesafe/conf", "reference.conf", //(these definitely need to be removed) "org/jboss/netty", //(not sure about this, can't imagine i actually need it though? If so try putting back in) "org/apache/curator", "org/apache/zookeeper")); //(not sure about these, can try putting them back in if needed) return Tuples._3T(f_tmp, f_p._2(), null); } else return Tuples._3T(f_p._1(), f_p._2(), fs); }
From source file:com.ikanow.aleph2.core.shared.utils.JarCacheUtils.java
License:Apache License
/** Moves a shared JAR into a local spot (if required) * @param library_bean/*from www. j a v a 2 s .c o m*/ * @param fs * @return either a basic message bean containing an error, or the fully qualified path of the cached JAR */ public static <M> CompletableFuture<Validation<BasicMessageBean, String>> getCachedJar( final String local_cached_jar_dir, final SharedLibraryBean library_bean, final IStorageService fs, final String handler_for_errors, final M msg_for_errors) { try { final FileContext dfs = fs.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get(); final FileContext lfs = fs.getUnderlyingPlatformDriver(FileContext.class, IStorageService.LOCAL_FS) .get(); final Path cached_jar_file = lfs .makeQualified(new Path(local_cached_jar_dir + "/" + buildCachedJarName(library_bean))); final Path original_jar_file = dfs.makeQualified(new Path(library_bean.path_name())); final FileStatus file_status = dfs.getFileStatus(original_jar_file); // (this will exception out if it doesn't exist, as it should) try { final FileStatus local_file_status = lfs.getFileStatus(cached_jar_file); // (this will exception in to case 2 if it doesn't exist) // if the local version exists then overwrite it if (file_status.getModificationTime() > local_file_status.getModificationTime()) { // (it gets kinda complicated here so just invalidate the entire classloader cache..) // TODO (ALEPH-12): add a coverage test for this ClassloaderUtils.clearCache(); lfs.util().copy(original_jar_file, cached_jar_file, false, true); } } catch (FileNotFoundException f) { // 2) if the local version doesn't exist then just copy the distributed file across // (note: don't need to do anything with the classloader cache here since the file doesn't exist so can't have a cache key) lfs.util().copy(original_jar_file, cached_jar_file); } return CompletableFuture.completedFuture(Validation.success(cached_jar_file.toString())); } catch (Throwable e) { return CompletableFuture.completedFuture( Validation.fail(SharedErrorUtils.buildErrorMessage(handler_for_errors, msg_for_errors, SharedErrorUtils.getLongForm(SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND, e, library_bean.path_name())))); } }
From source file:com.ikanow.aleph2.core.shared.utils.TestJarCacheUtils.java
License:Apache License
@Before public void setUpDependencies() throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IllegalArgumentException, IOException { final String temp_dir = System.getProperty("java.io.tmpdir") + File.separator; _globals = new GlobalPropertiesBean(temp_dir, temp_dir, temp_dir, temp_dir); _mock_hdfs = new MockStorageService(_globals); // Create a pretend "remote" file _test_file_path = temp_dir + "/test.jar"; final Path test_file_path = new Path(_test_file_path); final FileContext fs = _mock_hdfs.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()).get(); fs.create(test_file_path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); _test_file_time = fs.getFileStatus(test_file_path).getModificationTime(); }
From source file:com.ikanow.aleph2.core.shared.utils.TestJarCacheUtils.java
License:Apache License
@Test public void test_localFilePresentButOld() throws InterruptedException, ExecutionException, AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, IOException { final FileContext localfs = FileContext.getLocalFSFileContext(new Configuration()); String java_name = _globals.local_cached_jar_dir() + File.separator + "testX.cache.jar"; final String expected_cache_name = java_name.replace(File.separator, "/"); final Path expected_cache_path = localfs.makeQualified(new Path(expected_cache_name)); // Just make sure we've deleted the old file try {// w w w. ja v a 2s.com System.out.println("Deleted: " + new File(java_name).delete()); } catch (Exception e) { fail("Misc Error: " + e); } assertTrue("Remote file exists", new File(_test_file_path).exists()); assertFalse("Local file doesn't exist: " + java_name, new File(java_name).exists()); // Now create the file localfs.create(expected_cache_path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); localfs.setTimes(expected_cache_path, 0L, 0L); // check something has happened: assertEquals(0L, localfs.getFileStatus(expected_cache_path).getModificationTime()); assertNotEquals(0L, _test_file_time); // Now run the test routine final SharedLibraryBean library_bean = BeanTemplateUtils.build(SharedLibraryBean.class) .with(SharedLibraryBean::path_name, _test_file_path).with(SharedLibraryBean::_id, "testX").done() .get(); final Validation<BasicMessageBean, String> ret_val_1 = JarCacheUtils.getCachedJar( _globals.local_cached_jar_dir(), library_bean, _mock_hdfs, "testX", new TestMessageBean()).get(); assertEquals(expected_cache_path.toString(), ret_val_1.success()); assertTrue("Local file still exists", new File(expected_cache_name).exists()); assertTrue("File time should have been updated", localfs.getFileStatus(expected_cache_path).getModificationTime() >= _test_file_time); }
From source file:com.ikanow.aleph2.core.shared.utils.TestJarCacheUtils.java
License:Apache License
@Test public void test_localFilePresentAndNew() throws InterruptedException, ExecutionException, AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, IOException { final FileContext localfs = FileContext.getLocalFSFileContext(new Configuration()); final String expected_cache_name = _globals.local_cached_jar_dir().replace(File.separator, "/") + "test1.cache.jar"; final Path expected_cache_path = localfs.makeQualified(new Path(expected_cache_name)); // Just make sure we've deleted the old file try {//w ww . jav a 2 s . co m new File(expected_cache_name).delete(); } catch (Exception e) { } assertTrue("Remote file exists", new File(_test_file_path).exists()); assertFalse("Local file doesn't exist", new File(expected_cache_name).exists()); // Now create the file localfs.create(expected_cache_path, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); localfs.setTimes(expected_cache_path, _test_file_time + 10000, _test_file_time + 10000); // check something has happened: assertEquals(_test_file_time + 10000, localfs.getFileStatus(expected_cache_path).getModificationTime()); // Now run the test routine final SharedLibraryBean library_bean = BeanTemplateUtils.build(SharedLibraryBean.class) .with(SharedLibraryBean::path_name, _test_file_path).with(SharedLibraryBean::_id, "test1").done() .get(); final Validation<BasicMessageBean, String> ret_val_1 = JarCacheUtils.getCachedJar( _globals.local_cached_jar_dir(), library_bean, _mock_hdfs, "test1", new TestMessageBean()).get(); assertEquals(expected_cache_path.toString(), ret_val_1.success()); assertTrue("Local file still exists", new File(expected_cache_name).exists()); assertEquals(localfs.getFileStatus(expected_cache_path).getModificationTime(), _test_file_time + 10000); }
From source file:com.ikanow.aleph2.management_db.services.DataBucketCrudService.java
License:Apache License
/** Check if bucket exists (or the path within the bucket if "file" optional specified * @param to_check//from ww w .j a v a2 s .c o m * @param storage_service * @param file - the file path in the bucket (checks bucket root path if left blank) * @return * @throws Exception */ public static boolean doesBucketPathExist(final DataBucketBean to_check, final IStorageService storage_service, final Optional<String> file) throws Exception { final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty()) .get(); final String bucket_root = storage_service.getBucketRootPath() + "/" + to_check.full_name() + IStorageService.BUCKET_SUFFIX; try { dfs.getFileStatus(new Path(bucket_root + file.map(s -> "/" + s).orElse(""))); return true; } catch (FileNotFoundException fe) { return false; } }