List of usage examples for org.apache.hadoop.fs FileContext getLocalFSFileContext
public static FileContext getLocalFSFileContext() throws UnsupportedFileSystemException
From source file:org.apache.tajo.worker.DeletionService.java
License:Apache License
static final FileContext getLfs() { try {/* w ww . j a va 2 s .c o m*/ return FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
From source file:org.apache.tez.dag.app.launcher.TestTezLocalCacheManager.java
License:Apache License
private static LocalResource createFile(String content) throws IOException { FileContext fs = FileContext.getLocalFSFileContext(); java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt"); File temp = tempFile.toFile(); temp.deleteOnExit();//ww w .j a va2s.c o m Path p = new Path("file:///" + tempFile.toAbsolutePath().toString()); Files.write(tempFile, content.getBytes()); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p); ret.setResource(yarnUrlFromPath); ret.setSize(content.getBytes().length); ret.setType(LocalResourceType.FILE); ret.setVisibility(LocalResourceVisibility.PRIVATE); ret.setTimestamp(fs.getFileStatus(p).getModificationTime()); return ret; }
From source file:org.apache.tez.dag.app.launcher.TezLocalCacheManager.java
License:Apache License
public TezLocalCacheManager(Map<String, LocalResource> resources, Configuration conf) throws IOException { this.ugi = UserGroupInformation.getCurrentUser(); this.fileContext = FileContext.getLocalFSFileContext(); this.resources = resources; this.conf = conf; this.tempDir = Files.createTempDirectory(Paths.get("."), "tez-local-cache"); }
From source file:org.apache.tez.service.MiniTezTestServiceCluster.java
License:Apache License
private MiniTezTestServiceCluster(String clusterName, int numExecutorsPerService, long availableMemory, int numLocalDirs) { super(clusterName + "_TezTestServerCluster"); Preconditions.checkArgument(numExecutorsPerService > 0); Preconditions.checkArgument(availableMemory > 0); Preconditions.checkArgument(numLocalDirs > 0); String clusterNameTrimmed = clusterName.replace("$", "") + "_TezTestServerCluster"; File targetWorkDir = new File("target", clusterNameTrimmed); try {/*from w w w . j a v a 2 s . c om*/ FileContext.getLocalFSFileContext().delete(new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { LOG.warn("Could not cleanup test workDir: " + targetWorkDir, e); throw new RuntimeException("Could not cleanup test workDir: " + targetWorkDir, e); } if (Shell.WINDOWS) { // The test working directory can exceed the maximum path length supported // by some Windows APIs and cmd.exe (260 characters). To work around this, // create a symlink in temporary storage with a much shorter path, // targeting the full path to the test working directory. Then, use the // symlink as the test working directory. String targetPath = targetWorkDir.getAbsolutePath(); File link = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); String linkPath = link.getAbsolutePath(); try { FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); } catch (IOException e) { throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e); } // Guarantee target exists before creating symlink. targetWorkDir.mkdirs(); Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( Shell.getSymlinkCommand(targetPath, linkPath)); try { shexec.execute(); } catch (IOException e) { throw new YarnRuntimeException( String.format("failed to create symlink from %s to %s, shell output: %s", linkPath, targetPath, shexec.getOutput()), e); } this.testWorkDir = link; } else { this.testWorkDir = targetWorkDir; } this.numExecutorsPerService = numExecutorsPerService; this.availableMemory = availableMemory; // Setup Local Dirs localDirs = new String[numLocalDirs]; for (int i = 0; i < numLocalDirs; i++) { File f = new File(testWorkDir, "localDir"); f.mkdirs(); LOG.info("Created localDir: " + f.getAbsolutePath()); localDirs[i] = f.getAbsolutePath(); } }
From source file:org.kitesdk.examples.Main.java
License:Apache License
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: java -jar zips-1.jar <zips.json> <out.sequence>"); System.exit(1);/*from w w w . ja v a 2 s . co m*/ } SequenceFileInputFormat in; File file = new File(args[0]); if (!file.exists() || !file.canRead()) { System.err.println("Cannot read " + file); } Schema schema = ReflectData.get().getSchema(ZipCode.class); JSONFileReader<ZipCode> reader = new JSONFileReader<ZipCode>(new FileInputStream(file), schema, ZipCode.class); reader.initialize(); FileContext context = FileContext.getLocalFSFileContext(); SequenceFile.Writer writer = SequenceFile.createWriter(context, new Configuration(), new Path(args[1]), NullWritable.class, ZipCode.class, SequenceFile.CompressionType.NONE, null, new SequenceFile.Metadata(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)); for (ZipCode zip : reader) { writer.append(NullWritable.get(), zip); } writer.close(); }
From source file:org.springframework.yarn.test.MiniYARNCluster.java
License:Apache License
/** * @param testName/*from w w w . j a v a 2 s.c o m*/ * name of the test * @param noOfNodeManagers * the number of node managers in the cluster * @param numLocalDirs * the number of nm-local-dirs per nodemanager * @param numLogDirs * the number of nm-log-dirs per nodemanager */ public MiniYARNCluster(String testName, int noOfNodeManagers, int numLocalDirs, int numLogDirs) { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; String testSubDir = testName.replace("$", ""); File targetWorkDir = new File("target", testSubDir); try { FileContext.getLocalFSFileContext().delete(new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { LOG.warn("COULD NOT CLEANUP", e); throw new YarnException("could not cleanup test dir", e); } if (Shell.WINDOWS) { // The test working directory can exceed the maximum path length // supported // by some Windows APIs and cmd.exe (260 characters). To work around // this, // create a symlink in temporary storage with a much shorter path, // targeting the full path to the test working directory. Then, use // the // symlink as the test working directory. String targetPath = targetWorkDir.getAbsolutePath(); File link = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); String linkPath = link.getAbsolutePath(); try { FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); } catch (IOException e) { throw new YarnException("could not cleanup symlink: " + linkPath, e); } // Guarantee target exists before creating symlink. targetWorkDir.mkdirs(); ShellCommandExecutor shexec = new ShellCommandExecutor(Shell.getSymlinkCommand(targetPath, linkPath)); try { shexec.execute(); } catch (IOException e) { throw new YarnException(String.format("failed to create symlink from %s to %s, shell output: %s", linkPath, targetPath, shexec.getOutput()), e); } this.testWorkDir = link; } else { this.testWorkDir = targetWorkDir; } resourceManagerWrapper = new ResourceManagerWrapper(); addService(resourceManagerWrapper); nodeManagers = new CustomNodeManager[noOfNodeManagers]; for (int index = 0; index < noOfNodeManagers; index++) { addService(new NodeManagerWrapper(index)); nodeManagers[index] = new CustomNodeManager(); } }
From source file:oz.hadoop.yarn.test.cluster.InJvmContainerExecutor.java
License:Apache License
/** * *///from w w w . ja va2 s . co m public InJvmContainerExecutor() { try { this.fc = FileContext.getLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new IllegalStateException(e); } }
From source file:oz.hadoop.yarn.test.cluster.MiniYarnCluster.java
License:Apache License
/** * * @param clusterName/* ww w. j a va 2 s . c o m*/ */ private void prepareScriptExecutionEnv(String clusterName) { String testSubDir = clusterName.replace("$", ""); File targetWorkDir = new File("target", testSubDir); try { FileContext.getLocalFSFileContext().delete(new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { logger.warn("COULD NOT CLEANUP", e); throw new YarnRuntimeException("could not cleanup test dir: " + e, e); } if (Shell.WINDOWS) { // The test working directory can exceed the maximum path length // supported // by some Windows APIs and cmd.exe (260 characters). To work around // this, // create a symlink in temporary storage with a much shorter path, // targeting the full path to the test working directory. Then, use // the // symlink as the test working directory. String targetPath = targetWorkDir.getAbsolutePath(); File link = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); String linkPath = link.getAbsolutePath(); try { FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); } catch (IOException e) { throw new YarnRuntimeException("could not cleanup symlink: " + linkPath, e); } // Guarantee target exists before creating symlink. targetWorkDir.mkdirs(); ShellCommandExecutor shexec = new ShellCommandExecutor(Shell.getSymlinkCommand(targetPath, linkPath)); try { shexec.execute(); } catch (IOException e) { throw new YarnRuntimeException( String.format("failed to create symlink from %s to %s, shell output: %s", linkPath, targetPath, shexec.getOutput()), e); } this.testWorkDir = link; } else { this.testWorkDir = targetWorkDir; } }