Example usage for org.apache.hadoop.fs FileUtil fullyDelete

List of usage examples for org.apache.hadoop.fs FileUtil fullyDelete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil fullyDelete.

Prototype

public static boolean fullyDelete(final File dir) 

Source Link

Document

Delete a directory and all its contents.

Usage

From source file:com.cloudera.science.quince.LoadVariantsToolIT.java

License:Open Source License

@Test
public void testSmallVCF() throws Exception {

    String baseDir = "target/datasets";

    FileUtil.fullyDelete(new File(baseDir));

    String sampleGroup = "sample1";
    String input = "datasets/variants_vcf/small.vcf";
    String output = "dataset:file:target/datasets/variants_flat_locuspart";

    int exitCode = tool.run(new String[] { "--sample-group", sampleGroup, input, output });

    assertEquals(0, exitCode);/* w  w w.  ja v  a2 s  . c  o  m*/
    File partition = new File(baseDir, "variants_flat_locuspart/chr=1/pos=0/sample_group=sample1");
    assertTrue(partition.exists());

    File[] dataFiles = partition.listFiles(new FileFilter() {
        @Override
        public boolean accept(File pathname) {
            return !pathname.getName().startsWith(".");
        }
    });

    assertEquals(1, dataFiles.length);
    assertTrue(dataFiles[0].getName().endsWith(".parquet"));
}

From source file:com.datatorrent.benchmark.spillable.SpillableBenchmarkAppTester.java

License:Apache License

@Before
public void before() {
    FileUtil.fullyDelete(new File(basePath));
}

From source file:com.datatorrent.common.util.AsyncFSStorageAgent.java

License:Apache License

public void copyToHDFS(final int operatorId, final long windowId) throws IOException {
    if (this.localBasePath == null) {
        throw new AssertionError("save() was not called before copyToHDFS");
    }//w  ww. j av a  2 s. c om
    String operatorIdStr = String.valueOf(operatorId);
    File directory = new File(localBasePath, operatorIdStr);
    String window = Long.toHexString(windowId);
    Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
    File srcFile = new File(directory, String.valueOf(windowId));
    FSDataOutputStream stream = null;
    boolean stateSaved = false;
    try {
        // Create the temporary file with OverWrite option to avoid dangling lease issue and avoid exception if file already exists
        stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                Options.CreateOpts.CreateParent.createParent());
        InputStream in = null;
        try {
            in = new FileInputStream(srcFile);
            IOUtils.copyBytes(in, stream, conf, false);
        } finally {
            IOUtils.closeStream(in);
        }
        stateSaved = true;
    } catch (Throwable t) {
        logger.debug("while saving {} {}", operatorId, window, t);
        stateSaved = false;
        throw Throwables.propagate(t);
    } finally {
        try {
            if (stream != null) {
                stream.close();
            }
        } catch (IOException ie) {
            stateSaved = false;
            throw new RuntimeException(ie);
        } finally {
            if (stateSaved) {
                fileContext.rename(lPath,
                        new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
                        Options.Rename.OVERWRITE);
            }
            FileUtil.fullyDelete(srcFile);
        }
    }
}

From source file:com.ifeng.hadoop.thinker.LogDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }/* w w  w.  ja  v a 2  s  . c  o  m*/

    Job job = Job.getInstance(super.getConf());
    job.setJarByClass(getClass());

    FileUtil.fullyDelete(new File(args[1]));

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(LogMapper.class);
    job.setReducerClass(LogReducer.class);
    job.setCombinerClass(LogReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.ifeng.hadoop.thinker.LogLocalDriver.java

License:Apache License

public static void main(String[] args) throws Exception {

    String input = "src/test/resources/input/*.txt";
    String output = "src/test/resources/output";

    args = new String[] { input, output };

    FileUtil.fullyDelete(new File(output));

    int exitCode = ToolRunner.run(new LogLocalDriver(), args);
    System.exit(exitCode);//from  w  ww . j  a  v  a 2s  . com
}

From source file:com.ifeng.ipserver.IPServerLogParseLocalDriver.java

License:Apache License

public static void main(String[] args) throws Exception {
    String input = "src/test/resources/input/ipserver/mini.txt";
    String output = "src/test/resources/output/ipserver";

    if (args == null || args.length != 2) {
        args = new String[] { input, output };
    }/*from   ww  w .  j a v a 2 s. co  m*/

    FileUtil.fullyDelete(new File(output));

    int exitCode = ToolRunner.run(new IPServerLogParseLocalDriver(), args);
    System.exit(exitCode);
}

From source file:com.ifeng.logparser.NginxLogDriver.java

License:Apache License

public static void main(String[] args) throws Exception {
    //String input = "src/test/resources/data";
    String input = "/home/lhfei/app_tmp/data/";
    String output = "src/test/resources/output";

    if (args == null || args.length != 2) {
        args = new String[] { input, output };
    }/*  w  w  w.  ja va 2s  . c o  m*/

    FileUtil.fullyDelete(new File(output));

    int exitCode = ToolRunner.run(new NginxLogDriver(), args);
    System.exit(exitCode);
}

From source file:com.ifeng.vdn.iparea.parser.IPAreaLocalDriver.java

License:Apache License

public static void main(String[] args) throws Exception {

    String input = "/home/lhfei/app_tmp/iparea.txt";
    String output = "/home/lhfei/app_tmp/iparea/";

    //String input = "/home/lhfei/app_tmp/vdnlog/result_ALL.txt";
    //String output = "/home/lhfei/app_tmp/vdnlog/output/";

    if (args == null || args.length != 2) {
        args = new String[] { input, output };
    }//  ww  w .  ja  va2 s  .  co  m

    FileUtil.fullyDelete(new File(output));

    int exitCode = ToolRunner.run(new IPAreaLocalDriver(), args);
    System.exit(exitCode);
}

From source file:com.ifeng.vdn.loggroup.mapper.VideologGroupDriver.java

License:Apache License

public static void main(String[] args) throws Exception {
    String input = "src/test/resources/input/videolog/0000.sta";
    String output = "src/test/resources/output/log";

    /*String input = "/home/lhfei/app_tmp/2015-03-08.log.gz";
    String output = "src/test/resources/output/log8";
    *//* ww  w .ja va  2s  .  c o m*/

    if (args == null || args.length != 2) {
        args = new String[] { input, output };
    }

    FileUtil.fullyDelete(new File(output));

    int exitCode = ToolRunner.run(new VideologGroupDriver(), args);
    System.exit(exitCode);
}

From source file:com.ifeng.vdn.logparser.mapper.VideoLogDriver.java

License:Apache License

public static void main(String[] args) throws Exception {
    String input = "src/test/resources/input/videolog/0000.sta";
    String output = "src/test/resources/output/log";

    /*String input = "/home/lhfei/app_tmp/2015-03-05.log.gz";
    String output = "src/test/resources/output/log2";*/
    if (args == null || args.length != 2) {
        args = new String[] { input, output };
    }/*  w ww.j  av  a2s  . c o  m*/

    FileUtil.fullyDelete(new File(output));

    int exitCode = ToolRunner.run(new VideoLogDriver(), args);
    System.exit(exitCode);
}