List of usage examples for org.apache.hadoop.fs FsShell run
@Override public int run(String argv[]) throws Exception
From source file:com.pinterest.hdfsbackup.distcp.DistCp.java
License:Apache License
/** Delete the dst files/dirs which do not exist in src */ static private void deleteNonexisting(FileSystem dstfs, FileStatus dstroot, Path dstsorted, FileSystem jobfs, Path jobdir, JobConf jobconf, Configuration conf) throws IOException { if (!dstroot.isDir()) { throw new IOException("dst must be a directory when option " + Options.DELETE.cmd + " is set, but dst (= " + dstroot.getPath() + ") is not a directory."); }/* w w w .j av a 2 s . c o m*/ //write dst lsr results final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr"); final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf, dstlsr, Text.class, FileStatus.class, SequenceFile.CompressionType.NONE); try { //do lsr to get all file statuses in dstroot final Stack<FileStatus> lsrstack = new Stack<FileStatus>(); for (lsrstack.push(dstroot); !lsrstack.isEmpty();) { final FileStatus status = lsrstack.pop(); if (status.isDir()) { for (FileStatus child : dstfs.listStatus(status.getPath())) { String relative = makeRelative(dstroot.getPath(), child.getPath()); writer.append(new Text(relative), child); lsrstack.push(child); } } } } finally { checkAndClose(writer); } //sort lsr results final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted"); SequenceFile.Sorter sorter = new SequenceFile.Sorter(jobfs, new Text.Comparator(), Text.class, FileStatus.class, jobconf); sorter.sort(dstlsr, sortedlsr); //compare lsr list and dst list SequenceFile.Reader lsrin = null; SequenceFile.Reader dstin = null; try { lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf); dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf); //compare sorted lsr list and sorted dst list final Text lsrpath = new Text(); final FileStatus lsrstatus = new FileStatus(); final Text dstpath = new Text(); final Text dstfrom = new Text(); final FsShell shell = new FsShell(conf); final String[] shellargs = { "-rmr", null }; boolean hasnext = dstin.next(dstpath, dstfrom); for (; lsrin.next(lsrpath, lsrstatus);) { int dst_cmp_lsr = dstpath.compareTo(lsrpath); for (; hasnext && dst_cmp_lsr < 0;) { hasnext = dstin.next(dstpath, dstfrom); dst_cmp_lsr = dstpath.compareTo(lsrpath); } if (dst_cmp_lsr == 0) { //lsrpath exists in dst, skip it hasnext = dstin.next(dstpath, dstfrom); } else { //lsrpath does not exist, delete it String s = new Path(dstroot.getPath(), lsrpath.toString()).toString(); if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) { shellargs[1] = s; int r = 0; try { r = shell.run(shellargs); } catch (Exception e) { throw new IOException("Exception from shell.", e); } if (r != 0) { throw new IOException( "\"" + shellargs[0] + " " + shellargs[1] + "\" returns non-zero value " + r); } } } } } finally { checkAndClose(lsrin); checkAndClose(dstin); } }
From source file:com.scaleunlimited.cascading.DistCp.java
License:Apache License
/** Delete the dst files/dirs which do not exist in src */ static private void deleteNonexisting(FileSystem dstfs, FileStatus dstroot, Path dstsorted, FileSystem jobfs, Path jobdir, JobConf jobconf, Configuration conf) throws IOException { if (!dstroot.isDir()) { throw new IOException("dst must be a directory when option " + Options.DELETE.cmd + " is set, but dst (= " + dstroot.getPath() + ") is not a directory."); }/* www.j a v a 2 s . c o m*/ //write dst lsr results final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr"); final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf, dstlsr, Text.class, FileStatus.class, SequenceFile.CompressionType.NONE); try { //do lsr to get all file statuses in dstroot final Stack<FileStatus> lsrstack = new Stack<FileStatus>(); for (lsrstack.push(dstroot); !lsrstack.isEmpty();) { final FileStatus status = lsrstack.pop(); if (status.isDir()) { for (FileStatus child : dstfs.listStatus(status.getPath())) { String relative = makeRelative(dstroot.getPath(), child.getPath()); writer.append(new Text(relative), child); lsrstack.push(child); } } } } finally { checkAndClose(writer); } //sort lsr results final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted"); SequenceFile.Sorter sorter = new SequenceFile.Sorter(jobfs, new Text.Comparator(), Text.class, FileStatus.class, jobconf); sorter.sort(dstlsr, sortedlsr); //compare lsr list and dst list SequenceFile.Reader lsrin = null; SequenceFile.Reader dstin = null; try { lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf); dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf); //compare sorted lsr list and sorted dst list final Text lsrpath = new Text(); final FileStatus lsrstatus = new FileStatus(); final Text dstpath = new Text(); final Text dstfrom = new Text(); final FsShell shell = new FsShell(conf); final String[] shellargs = { "-rmr", null }; boolean hasnext = dstin.next(dstpath, dstfrom); for (; lsrin.next(lsrpath, lsrstatus);) { int dst_cmp_lsr = dstpath.compareTo(lsrpath); for (; hasnext && dst_cmp_lsr < 0;) { hasnext = dstin.next(dstpath, dstfrom); dst_cmp_lsr = dstpath.compareTo(lsrpath); } if (dst_cmp_lsr == 0) { //lsrpath exists in dst, skip it hasnext = dstin.next(dstpath, dstfrom); } else { //lsrpath does not exist, delete it String s = new Path(dstroot.getPath(), lsrpath.toString()).toString(); if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) { shellargs[1] = s; int r = 0; try { r = shell.run(shellargs); } catch (Exception e) { throw new IOException("Exception from shell.", e); } if (r != 0) { throw new IOException( "\"" + shellargs[0] + " " + shellargs[1] + "\" returns non-zero value " + r); } } } } } finally { checkAndClose(lsrin); checkAndClose(dstin); } }
From source file:fr.ens.biologie.genomique.eoulsan.modules.mgmt.hadoop.DistCp.java
License:LGPL
/** Delete the dst files/dirs which do not exist in src */ static private void deleteNonexisting(final FileSystem dstfs, final FileStatus dstroot, final Path dstsorted, final FileSystem jobfs, final Path jobdir, final JobConf jobconf, final Configuration conf) throws IOException { if (!dstroot.isDir()) { throw new IOException("dst must be a directory when option " + Options.DELETE.cmd + " is set, but dst (= " + dstroot.getPath() + ") is not a directory."); }//from ww w . j a v a 2 s . com // write dst lsr results final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr"); final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf, dstlsr, Text.class, dstroot.getClass(), SequenceFile.CompressionType.NONE); try { // do lsr to get all file statuses in dstroot final Stack<FileStatus> lsrstack = new Stack<>(); for (lsrstack.push(dstroot); !lsrstack.isEmpty();) { final FileStatus status = lsrstack.pop(); if (status.isDir()) { for (FileStatus child : dstfs.listStatus(status.getPath())) { String relative = makeRelative(dstroot.getPath(), child.getPath()); writer.append(new Text(relative), child); lsrstack.push(child); } } } } finally { checkAndClose(writer); } // sort lsr results final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted"); SequenceFile.Sorter sorter = new SequenceFile.Sorter(jobfs, new Text.Comparator(), Text.class, FileStatus.class, jobconf); sorter.sort(dstlsr, sortedlsr); // compare lsr list and dst list SequenceFile.Reader lsrin = null; SequenceFile.Reader dstin = null; try { lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf); dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf); // compare sorted lsr list and sorted dst list final Text lsrpath = new Text(); final FileStatus lsrstatus = new FileStatus(); final Text dstpath = new Text(); final Text dstfrom = new Text(); final FsShell shell = new FsShell(conf); final String[] shellargs = { "-rmr", null }; boolean hasnext = dstin.next(dstpath, dstfrom); for (; lsrin.next(lsrpath, lsrstatus);) { int dst_cmp_lsr = dstpath.compareTo(lsrpath); for (; hasnext && dst_cmp_lsr < 0;) { hasnext = dstin.next(dstpath, dstfrom); dst_cmp_lsr = dstpath.compareTo(lsrpath); } if (dst_cmp_lsr == 0) { // lsrpath exists in dst, skip it hasnext = dstin.next(dstpath, dstfrom); } else { // lsrpath does not exist, delete it String s = new Path(dstroot.getPath(), lsrpath.toString()).toString(); if (shellargs[1] == null || !isAncestorPath(shellargs[1], s)) { shellargs[1] = s; int r = 0; try { r = shell.run(shellargs); } catch (Exception e) { throw new IOException("Exception from shell.", e); } if (r != 0) { throw new IOException( "\"" + shellargs[0] + " " + shellargs[1] + "\" returns non-zero value " + r); } } } } } finally { checkAndClose(lsrin); checkAndClose(dstin); } }
From source file:io.aos.hadoop.AosFsShell.java
License:Apache License
public static void main(String... args) throws Exception { FsShell fsShell = new FsShell(new Configuration()); fsShell.run(new String[] { "-ls", "/" }); // fsShell.run(new String[]{"-ls /"}); // fsShell.run(new String[]{"-ls", "hdfs://"}); }
From source file:org.apache.accumulo.examples.mapreduce.bulk.BulkIngestExample.java
License:Apache License
@Override public int run(String[] args) { Opts opts = new Opts(); opts.parseArgs(BulkIngestExample.class.getName(), args); Configuration conf = getConf(); PrintStream out = null;//from w ww. ja v a 2s . c o m try { Job job = Job.getInstance(conf); job.setJobName("bulk ingest example"); job.setJarByClass(this.getClass()); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MapClass.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setReducerClass(ReduceClass.class); job.setOutputFormatClass(AccumuloFileOutputFormat.class); opts.setAccumuloConfigs(job); Connector connector = opts.getConnector(); TextInputFormat.setInputPaths(job, new Path(opts.inputDir)); AccumuloFileOutputFormat.setOutputPath(job, new Path(opts.workDir + "/files")); FileSystem fs = FileSystem.get(conf); out = new PrintStream(new BufferedOutputStream(fs.create(new Path(opts.workDir + "/splits.txt")))); Collection<Text> splits = connector.tableOperations().listSplits(opts.getTableName(), 100); for (Text split : splits) out.println(Base64.getEncoder().encodeToString(TextUtil.getBytes(split))); job.setNumReduceTasks(splits.size() + 1); out.close(); job.setPartitionerClass(RangePartitioner.class); RangePartitioner.setSplitFile(job, opts.workDir + "/splits.txt"); job.waitForCompletion(true); Path failures = new Path(opts.workDir, "failures"); fs.delete(failures, true); fs.mkdirs(new Path(opts.workDir, "failures")); // With HDFS permissions on, we need to make sure the Accumulo user can read/move the rfiles FsShell fsShell = new FsShell(conf); fsShell.run(new String[] { "-chmod", "-R", "777", opts.workDir }); connector.tableOperations().importDirectory(opts.getTableName(), opts.workDir + "/files", opts.workDir + "/failures", false); } catch (Exception e) { throw new RuntimeException(e); } finally { if (out != null) out.close(); } return 0; }
From source file:org.apache.accumulo.test.BulkImportSequentialRowsIT.java
License:Apache License
@Test public void testBulkImportFailure() throws Exception { String tableName = getUniqueNames(1)[0]; TableOperations to = getConnector().tableOperations(); to.create(tableName);/*from w ww . j a v a2 s . c o m*/ FileSystem fs = getFileSystem(); Path rootPath = new Path(fs.makeQualified(getUsableDir()), getClass().getSimpleName()); log.info("Writing to {}", rootPath); if (fs.exists(rootPath)) { assertTrue(fs.delete(rootPath, true)); } assertTrue(fs.mkdirs(rootPath)); Path bulk = new Path(rootPath, "bulk"); log.info("bulk: {}", bulk); assertTrue(fs.mkdirs(bulk)); Path err = new Path(rootPath, "err"); log.info("err: {}", err); assertTrue(fs.mkdirs(bulk)); assertTrue(fs.mkdirs(err)); Path rfile = new Path(bulk, "file.rf"); log.info("Generating RFile {}", rfile.toUri().toString()); GenerateSequentialRFile.main(new String[] { "-f", rfile.toUri().toString(), "-nr", Long.toString(NR), "-nv", Long.toString(NV) }); assertTrue("Expected that " + rfile + " exists, but it does not", fs.exists(rfile)); FsShell fsShell = new FsShell(fs.getConf()); assertEquals("Failed to chmod " + rootPath, 0, fsShell.run(new String[] { "-chmod", "-R", "777", rootPath.toString() })); // Add some splits to.addSplits(tableName, getSplits()); // Then import a single rfile to all the tablets, hoping that we get a failure to import because of the balancer moving tablets around // and then we get to verify that the bug is actually fixed. to.importDirectory(tableName, bulk.toString(), err.toString(), false); // The bug is that some tablets don't get imported into. assertEquals(NR * NV, Iterables.size(getConnector().createScanner(tableName, Authorizations.EMPTY))); }
From source file:org.apache.kylin.engine.spark.SparkCubing.java
License:Apache License
private void bulkLoadHFile(String cubeName, String segmentId, String hfileLocation) throws Exception { final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); final Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration(); FsShell shell = new FsShell(hbaseConf); try {// w w w .j av a 2 s.c o m shell.run(new String[] { "-chmod", "-R", "777", hfileLocation }); } catch (Exception e) { logger.error("Couldnt change the file permissions ", e); throw new IOException(e); } String[] newArgs = new String[2]; newArgs[0] = hfileLocation; newArgs[1] = cubeSegment.getStorageLocationIdentifier(); int ret = ToolRunner.run(new LoadIncrementalHFiles(hbaseConf), newArgs); System.out.println("incremental load result:" + ret); cubeSegment.setStatus(SegmentStatusEnum.READY); try { CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance); cubeInstance.setStatus(RealizationStatusEnum.READY); cubeSegment.setStatus(SegmentStatusEnum.READY); cubeBuilder.setToUpdateSegs(cubeSegment); CubeManager.getInstance(kylinConfig).updateCube(cubeBuilder); } catch (IOException e) { throw new RuntimeException("Failed to deal with the request: " + e.getLocalizedMessage()); } }
From source file:org.apache.kylin.storage.hbase.lookup.LookupTableHFilesBulkLoadJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_TABLE_NAME); options.addOption(OPTION_CUBING_JOB_ID); options.addOption(OPTION_LOOKUP_SNAPSHOT_ID); parseOptions(options, args);/*from w ww . j a v a2s. c om*/ String tableName = getOptionValue(OPTION_TABLE_NAME); String cubingJobID = getOptionValue(OPTION_CUBING_JOB_ID); String snapshotID = getOptionValue(OPTION_LOOKUP_SNAPSHOT_ID); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExecutableManager execMgr = ExecutableManager.getInstance(kylinConfig); DefaultChainedExecutable job = (DefaultChainedExecutable) execMgr.getJob(cubingJobID); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager .getInstance(kylinConfig); ExtTableSnapshotInfo snapshot = extTableSnapshotInfoManager.getSnapshot(tableName, snapshotID); long srcTableRowCnt = Long.parseLong( job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1")); logger.info("update table:{} snapshot row count:{}", tableName, srcTableRowCnt); snapshot.setRowCnt(srcTableRowCnt); snapshot.setLastBuildTime(System.currentTimeMillis()); extTableSnapshotInfoManager.updateSnapshot(snapshot); String hTableName = snapshot.getStorageLocationIdentifier(); // e.g // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/ // end with "/" String input = getOptionValue(OPTION_INPUT_PATH); Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); FsShell shell = new FsShell(conf); int exitCode = -1; int retryCount = 10; while (exitCode != 0 && retryCount >= 1) { exitCode = shell.run(new String[] { "-chmod", "-R", "777", input }); retryCount--; Thread.sleep(5000); } if (exitCode != 0) { logger.error("Failed to change the file permissions: {}", input); throw new IOException("Failed to change the file permissions: " + input); } String[] newArgs = new String[2]; newArgs[0] = input; newArgs[1] = hTableName; logger.debug("Start to run LoadIncrementalHFiles"); int ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs); logger.debug("End to run LoadIncrementalHFiles"); return ret; }
From source file:org.apache.kylin.storage.hbase.steps.BulkLoadJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args);/*ww w . j a va 2s . com*/ String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase(); // e.g // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/ // end with "/" String input = getOptionValue(OPTION_INPUT_PATH); Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); FsShell shell = new FsShell(conf); int exitCode = -1; int retryCount = 10; while (exitCode != 0 && retryCount >= 1) { exitCode = shell.run(new String[] { "-chmod", "-R", "777", input }); retryCount--; Thread.sleep(5000); } if (exitCode != 0) { logger.error("Failed to change the file permissions: " + input); throw new IOException("Failed to change the file permissions: " + input); } String[] newArgs = new String[2]; newArgs[0] = input; newArgs[1] = tableName; logger.debug("Start to run LoadIncrementalHFiles"); int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs); logger.debug("End to run LoadIncrementalHFiles"); return ret; }
From source file:org.apache.pig.scripting.Pig.java
License:Apache License
/** * Run a filesystem command. Any output from this command is written to * stdout or stderr as appropriate.//from www . j a va2 s . c o m * @param cmd Filesystem command to run along with its arguments as one * string. * @throws IOException */ public static int fs(String cmd) throws IOException { ScriptPigContext ctx = getScriptContext(); FsShell shell = new FsShell(ConfigurationUtil.toConfiguration(ctx.getPigContext().getProperties())); int code = -1; if (cmd != null) { String[] cmdTokens = cmd.split("\\s+"); if (!cmdTokens[0].startsWith("-")) cmdTokens[0] = "-" + cmdTokens[0]; try { code = shell.run(cmdTokens); } catch (Exception e) { throw new IOException("Run filesystem command failed", e); } } return code; }