List of usage examples for org.apache.hadoop.fs FileUtil fullyDelete
public static boolean fullyDelete(final File dir)
From source file:com.cloudera.science.quince.LoadVariantsToolIT.java
License:Open Source License
@Test public void testSmallVCF() throws Exception { String baseDir = "target/datasets"; FileUtil.fullyDelete(new File(baseDir)); String sampleGroup = "sample1"; String input = "datasets/variants_vcf/small.vcf"; String output = "dataset:file:target/datasets/variants_flat_locuspart"; int exitCode = tool.run(new String[] { "--sample-group", sampleGroup, input, output }); assertEquals(0, exitCode);/* w w w. ja v a2 s . c o m*/ File partition = new File(baseDir, "variants_flat_locuspart/chr=1/pos=0/sample_group=sample1"); assertTrue(partition.exists()); File[] dataFiles = partition.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return !pathname.getName().startsWith("."); } }); assertEquals(1, dataFiles.length); assertTrue(dataFiles[0].getName().endsWith(".parquet")); }
From source file:com.datatorrent.benchmark.spillable.SpillableBenchmarkAppTester.java
License:Apache License
@Before public void before() { FileUtil.fullyDelete(new File(basePath)); }
From source file:com.datatorrent.common.util.AsyncFSStorageAgent.java
License:Apache License
public void copyToHDFS(final int operatorId, final long windowId) throws IOException { if (this.localBasePath == null) { throw new AssertionError("save() was not called before copyToHDFS"); }//w ww. j av a 2 s. c om String operatorIdStr = String.valueOf(operatorId); File directory = new File(localBasePath, operatorIdStr); String window = Long.toHexString(windowId); Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE); File srcFile = new File(directory, String.valueOf(windowId)); FSDataOutputStream stream = null; boolean stateSaved = false; try { // Create the temporary file with OverWrite option to avoid dangling lease issue and avoid exception if file already exists stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent()); InputStream in = null; try { in = new FileInputStream(srcFile); IOUtils.copyBytes(in, stream, conf, false); } finally { IOUtils.closeStream(in); } stateSaved = true; } catch (Throwable t) { logger.debug("while saving {} {}", operatorId, window, t); stateSaved = false; throw Throwables.propagate(t); } finally { try { if (stream != null) { stream.close(); } } catch (IOException ie) { stateSaved = false; throw new RuntimeException(ie); } finally { if (stateSaved) { fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window), Options.Rename.OVERWRITE); } FileUtil.fullyDelete(srcFile); } } }
From source file:com.ifeng.hadoop.thinker.LogDriver.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName()); ToolRunner.printGenericCommandUsage(System.err); return -1; }/* w w w. ja v a 2 s . c o m*/ Job job = Job.getInstance(super.getConf()); job.setJarByClass(getClass()); FileUtil.fullyDelete(new File(args[1])); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setMapperClass(LogMapper.class); job.setReducerClass(LogReducer.class); job.setCombinerClass(LogReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.ifeng.hadoop.thinker.LogLocalDriver.java
License:Apache License
public static void main(String[] args) throws Exception { String input = "src/test/resources/input/*.txt"; String output = "src/test/resources/output"; args = new String[] { input, output }; FileUtil.fullyDelete(new File(output)); int exitCode = ToolRunner.run(new LogLocalDriver(), args); System.exit(exitCode);//from w ww . j a v a 2s . com }
From source file:com.ifeng.ipserver.IPServerLogParseLocalDriver.java
License:Apache License
public static void main(String[] args) throws Exception { String input = "src/test/resources/input/ipserver/mini.txt"; String output = "src/test/resources/output/ipserver"; if (args == null || args.length != 2) { args = new String[] { input, output }; }/*from ww w . j a v a 2 s. co m*/ FileUtil.fullyDelete(new File(output)); int exitCode = ToolRunner.run(new IPServerLogParseLocalDriver(), args); System.exit(exitCode); }
From source file:com.ifeng.logparser.NginxLogDriver.java
License:Apache License
public static void main(String[] args) throws Exception { //String input = "src/test/resources/data"; String input = "/home/lhfei/app_tmp/data/"; String output = "src/test/resources/output"; if (args == null || args.length != 2) { args = new String[] { input, output }; }/* w w w. ja va 2s . c o m*/ FileUtil.fullyDelete(new File(output)); int exitCode = ToolRunner.run(new NginxLogDriver(), args); System.exit(exitCode); }
From source file:com.ifeng.vdn.iparea.parser.IPAreaLocalDriver.java
License:Apache License
public static void main(String[] args) throws Exception { String input = "/home/lhfei/app_tmp/iparea.txt"; String output = "/home/lhfei/app_tmp/iparea/"; //String input = "/home/lhfei/app_tmp/vdnlog/result_ALL.txt"; //String output = "/home/lhfei/app_tmp/vdnlog/output/"; if (args == null || args.length != 2) { args = new String[] { input, output }; }// ww w . ja va2 s . co m FileUtil.fullyDelete(new File(output)); int exitCode = ToolRunner.run(new IPAreaLocalDriver(), args); System.exit(exitCode); }
From source file:com.ifeng.vdn.loggroup.mapper.VideologGroupDriver.java
License:Apache License
public static void main(String[] args) throws Exception { String input = "src/test/resources/input/videolog/0000.sta"; String output = "src/test/resources/output/log"; /*String input = "/home/lhfei/app_tmp/2015-03-08.log.gz"; String output = "src/test/resources/output/log8"; *//* ww w .ja va 2s . c o m*/ if (args == null || args.length != 2) { args = new String[] { input, output }; } FileUtil.fullyDelete(new File(output)); int exitCode = ToolRunner.run(new VideologGroupDriver(), args); System.exit(exitCode); }
From source file:com.ifeng.vdn.logparser.mapper.VideoLogDriver.java
License:Apache License
public static void main(String[] args) throws Exception { String input = "src/test/resources/input/videolog/0000.sta"; String output = "src/test/resources/output/log"; /*String input = "/home/lhfei/app_tmp/2015-03-05.log.gz"; String output = "src/test/resources/output/log2";*/ if (args == null || args.length != 2) { args = new String[] { input, output }; }/* w ww.j av a2s . c o m*/ FileUtil.fullyDelete(new File(output)); int exitCode = ToolRunner.run(new VideoLogDriver(), args); System.exit(exitCode); }