Example usage for org.apache.hadoop.fs FileUtil fullyDelete

List of usage examples for org.apache.hadoop.fs FileUtil fullyDelete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileUtil
  6. fullyDelete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil fullyDelete.

Prototype

public static boolean fullyDelete(final File dir)

Source Link

Document

Delete a directory and all its contents.

Usage

From source file:com.uber.hoodie.common.minicluster.ZookeeperTestService.java

License:Apache License

private void recreateDir(File dir, boolean clean) throws IOException {
    if (dir.exists() && clean) {
        FileUtil.fullyDelete(dir);
    } else if (dir.exists() && !clean) {
        // the directory's exist, and we don't want to clean, so exit
        return;//from  w w w. ja  v  a  2s  .  c  om
    }
    try {
        dir.mkdirs();
    } catch (SecurityException e) {
        throw new IOException("creating dir: " + dir, e);
    }
}

From source file:com.uber.hoodie.common.table.log.HoodieLogFormatAppendFailureTest.java

License:Apache License

@BeforeClass
public static void setUpClass() throws IOException {
    // NOTE : The MiniClusterDFS leaves behind the directory under which the cluster was created
    baseDir = new File("/tmp/" + UUID.randomUUID().toString());
    FileUtil.fullyDelete(baseDir);
    // Append is not supported in LocalFileSystem. HDFS needs to be setup.
    Configuration conf = new Configuration();
    // lower heartbeat interval for fast recognition of DN
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 3000);
    cluster = new MiniDFSCluster.Builder(conf).checkExitOnShutdown(true).numDataNodes(4).build();
}

From source file:com.uber.hoodie.common.table.log.HoodieLogFormatAppendFailureTest.java

License:Apache License

@AfterClass
public static void tearDownClass() {
    cluster.shutdown(true);/*from  w  w w . j a va2s . c o  m*/
    // Force clean up the directory under which the cluster was created
    FileUtil.fullyDelete(baseDir);
}

From source file:csc555.ebratt.depaul.edu.Test_RCWordCountDriver.java

License:Open Source License

/**
 * @throws java.lang.Exception//from   www .j a  v  a 2  s. c o m
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
    // delete base dfs directory
    LOG.info("About to delete directory: " + BASEDIR.toString());
    boolean deleted = FileUtil.fullyDelete(BASEDIR);
    LOG.info("Delete successful? " + deleted);

    // set mini dfs cluster base directory
    LOG.info("mini dfs cluster directory: " + BASEDIR.getAbsolutePath());
    CONF.set("hdfs.minidfs.basedir", BASEDIR.getAbsolutePath());
}

From source file:CSVUtils.CSVUtils.java

public static void createCSV(ArrayList<KeyValueTuple> data, String headers, String outPut) {
    File outputFile = new File(outPut);
    if (outputFile.exists()) {
        FileUtil.fullyDelete(outputFile);
    }/*from   w ww. ja  va2  s  . c  o m*/
    try {
        outputFile.getParentFile().mkdirs();
        outputFile.createNewFile();
    } catch (Exception ex) {
        System.out.println("File creation failed");
    }
    try {
        PrintWriter writer = new PrintWriter(outputFile);
        writer.println(headers);

        for (KeyValueTuple kvt : data) {
            writer.println(kvt.getKey() + "," + kvt.getValue());
        }
        writer.close();
    } catch (Exception ex) {
        System.out.println("Error Reading Data");
    }
}

From source file:CSVUtils.CSVUtils.java

public static void createCSVForSentiment(ArrayList<KeyValueTuple> data, String headers, String outPut) {
    File outputFile = new File(outPut);
    if (outputFile.exists()) {
        FileUtil.fullyDelete(outputFile);
    }/*w w  w  .  j  av  a 2  s  .co m*/
    try {
        outputFile.getParentFile().mkdirs();
        outputFile.createNewFile();
    } catch (Exception ex) {
        System.out.println("File creation failed");
    }
    try {
        PrintWriter writer = new PrintWriter(outputFile);
        writer.println(headers);

        for (KeyValueTuple kvt : data) {
            writer.println(kvt.getKey() + "," + kvt.getValue() + "," + getSentimentColor(kvt.getKey()));
        }
        writer.close();
    } catch (Exception ex) {
        System.out.println("Error Reading Data");
    }
}

From source file:CSVUtils.CSVUtils.java

public static void createCSVLanguageAsterPlot(ArrayList<KeyValueTuple> data, String headers, String outPut) {
    File outputFile = new File(outPut);
    if (outputFile.exists()) {
        FileUtil.fullyDelete(outputFile);
    }//w w  w  . j a v a2 s .  c om
    try {
        outputFile.getParentFile().mkdirs();
        outputFile.createNewFile();
    } catch (Exception ex) {
        System.out.println("File creation failed");
    }
    try {
        NumberFormat formatter = new DecimalFormat("#0.0000");
        long total = 0;
        for (KeyValueTuple kvt : data) {
            total = total + kvt.getValue();
        }

        long maxcount = data.get(0).getValue();
        double order = 0.0;
        int count = 0;

        PrintWriter writer = new PrintWriter(outputFile);
        writer.println(headers);

        for (KeyValueTuple kvt : data) {
            double weight = kvt.getValue() / (double) total;
            double score = (kvt.getValue() / (double) maxcount) * 100;
            //"label", "order", "score", "weight" ,"color", "count"

            String str = kvt.getKey().toUpperCase() + "," + formatter.format(order) + ","
                    + formatter.format(score) + "," + formatter.format(weight) + "," + get20Colors(count) + ","
                    + kvt.getValue();
            writer.println(str);
            order++;
            count++;
        }
        writer.close();
    } catch (Exception ex) {
        System.out.println("Error Reading Data");
    }
}

From source file:CSVUtils.CSVUtils.java

public static void createCSVForDeviceCount(ArrayList<KeyValueTuple> data, String headers, String outPut) {
    File outputFile = new File(outPut);
    if (outputFile.exists()) {
        FileUtil.fullyDelete(outputFile);
    }//from   w  w  w.  j  av a  2  s.co  m
    try {
        outputFile.getParentFile().mkdirs();
        outputFile.createNewFile();
    } catch (Exception ex) {
        System.out.println("File creation failed");
    }
    try {
        PrintWriter writer = new PrintWriter(outputFile);
        writer.println(headers);

        for (KeyValueTuple kvt : data) {
            String country = (kvt.getKey().split("-")[0]).trim();
            String deviceType = (kvt.getKey().split("-")[1]).trim();
            DeviceAndCountryUtil dc = findCountryPresent(country);
            if (dc == null) {
                dc = new DeviceAndCountryUtil();
                dc.setCountry(country);
                listOfCountries.add(dc);
                if (deviceType.equals("Android")) {
                    dc.setAndroid(kvt.getValue());
                } else if (deviceType.equals("ioS")) {
                    dc.setIoS(kvt.getValue());
                } else if (deviceType.equals("Web")) {
                    dc.setWeb(kvt.getValue());
                } else if (deviceType.equals("Others")) {
                    dc.setOthers(kvt.getValue());
                }
            } else {
                if (deviceType.equals("Android")) {
                    dc.setAndroid(kvt.getValue());
                } else if (deviceType.equals("ioS")) {
                    dc.setIoS(kvt.getValue());
                } else if (deviceType.equals("Web")) {
                    dc.setWeb(kvt.getValue());
                } else if (deviceType.equals("Others")) {
                    dc.setOthers(kvt.getValue());
                }
            }
        }

        for (DeviceAndCountryUtil dc : listOfCountries) {
            dc.formString(writer);
        }
        writer.close();
    } catch (Exception ex) {
        System.out.println("Error Reading Data");
    }
}

From source file:CSVUtils.CSVUtils.java

public static void createCSVForGenderTimezone(ArrayList<KeyValueTuple> data, String headers, String outPut) {
    File outputFile = new File(outPut);
    if (outputFile.exists()) {
        FileUtil.fullyDelete(outputFile);
    }//from  w w w .jav a2  s  . c  o m
    try {
        outputFile.getParentFile().mkdirs();
        outputFile.createNewFile();
    } catch (Exception ex) {
        System.out.println("File creation failed");
    }
    try {
        PrintWriter writer = new PrintWriter(outputFile);
        writer.println(headers);

        for (KeyValueTuple kvt : data) {
            String timezone = (kvt.getKey().split("\t")[0]).trim();
            String gender = (kvt.getKey().split("\t")[1]).trim();
            GenderPerTimezoneUtil gtpz = findGenderTimezonePresent(timezone);
            if (gtpz == null) {
                gtpz = new GenderPerTimezoneUtil();
                gtpz.setValue(timezone);
                listOfTimezonesGender.add(gtpz);
            }
            if (gender.equals("MALE")) {
                gtpz.setMale(kvt.getValue());
            } else if (gender.equals("FEMALE")) {
                gtpz.setFemale(kvt.getValue());
            }
        }

        for (GenderPerTimezoneUtil gtpz : listOfTimezonesGender) {
            gtpz.formString(writer);
        }
        writer.close();
    } catch (Exception ex) {
        System.out.println("Error Reading Data");
    }
}

From source file:CSVUtils.CSVUtils.java

public static void generateGeoJs(ArrayList<KeyValueTuple> data, String outPut) {
    File outputFile = new File(outPut);
    if (outputFile.exists()) {
        FileUtil.fullyDelete(outputFile);
    }/*  ww  w.  j a  v a 2  s.com*/
    try {
        outputFile.getParentFile().mkdirs();
        outputFile.createNewFile();
    } catch (Exception ex) {
        System.out.println("File creation failed");
    }
    try {
        PrintWriter writer = new PrintWriter(outputFile);
        writer.println("var statesData = {\"type\":\"FeatureCollection\",\"features\":[");
        for (int i = 0; i < data.size() - 1; i++) {
            writer.println(GeoCoordinates.getCoordinatesJS(data.get(i).getKey(), "Tweets",
                    String.valueOf(data.get(i).getValue())) + ",");
        }
        writer.println(GeoCoordinates.getCoordinatesJS(data.get(data.size() - 1).getKey(), "Tweets",
                String.valueOf(data.get(data.size() - 1).getValue()))); //Print last without comma
        writer.println("]};");
        writer.close();
    } catch (Exception ex) {
        System.out.println("Error Reading Data");
    }
}