Example usage for org.apache.hadoop.fs FileUtil fullyDelete

List of usage examples for org.apache.hadoop.fs FileUtil fullyDelete

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileUtil
  6. fullyDelete

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileUtil fullyDelete.

Prototype

public static boolean fullyDelete(final File dir)

Source Link

Document

Delete a directory and all its contents.

Usage

From source file:org.apache.hcatalog.pig.TestHCatLoaderStorer.java

License:Apache License

/**
 * Ensure Pig can read/write tinyint/smallint columns.
 *///  w w w .  j ava2  s . c om
@Test
public void testSmallTinyInt() throws Exception {

    String readTblName = "test_small_tiny_int";
    File dataDir = new File(TEST_DATA_DIR + "/testSmallTinyIntData");
    File dataFile = new File(dataDir, "testSmallTinyInt.tsv");

    String writeTblName = "test_small_tiny_int_write";
    File writeDataFile = new File(TEST_DATA_DIR, writeTblName + ".tsv");

    FileUtil.fullyDelete(dataDir); // Might not exist
    Assert.assertTrue(dataDir.mkdir());

    HcatTestUtils.createTestDataFile(dataFile.getAbsolutePath(),
            new String[] { String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE),
                    String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE) });

    // Create a table with smallint/tinyint columns, load data, and query from Hive.
    Assert.assertEquals(0, driver.run("drop table if exists " + readTblName).getResponseCode());
    Assert.assertEquals(0,
            driver.run("create external table " + readTblName + " (my_small_int smallint, my_tiny_int tinyint)"
                    + " row format delimited fields terminated by '\t' stored as textfile").getResponseCode());
    Assert.assertEquals(0,
            driver.run("load data local inpath '" + dataDir.getAbsolutePath() + "' into table " + readTblName)
                    .getResponseCode());

    PigServer server = new PigServer(ExecType.LOCAL);
    server.registerQuery("data = load '" + readTblName + "' using org.apache.hcatalog.pig.HCatLoader();");

    // Ensure Pig schema is correct.
    Schema schema = server.dumpSchema("data");
    Assert.assertEquals(2, schema.getFields().size());
    Assert.assertEquals("my_small_int", schema.getField(0).alias);
    Assert.assertEquals(DataType.INTEGER, schema.getField(0).type);
    Assert.assertEquals("my_tiny_int", schema.getField(1).alias);
    Assert.assertEquals(DataType.INTEGER, schema.getField(1).type);

    // Ensure Pig can read data correctly.
    Iterator<Tuple> it = server.openIterator("data");
    Tuple t = it.next();
    Assert.assertEquals(new Integer(Short.MIN_VALUE), t.get(0));
    Assert.assertEquals(new Integer(Byte.MIN_VALUE), t.get(1));
    t = it.next();
    Assert.assertEquals(new Integer(Short.MAX_VALUE), t.get(0));
    Assert.assertEquals(new Integer(Byte.MAX_VALUE), t.get(1));
    Assert.assertFalse(it.hasNext());

    // Ensure Pig can write correctly to smallint/tinyint columns. This means values within the
    // bounds of the column type are written, and values outside throw an exception.
    Assert.assertEquals(0, driver.run("drop table if exists " + writeTblName).getResponseCode());
    Assert.assertEquals(
            0, driver
                    .run("create table " + writeTblName
                            + " (my_small_int smallint, my_tiny_int tinyint) stored as rcfile")
                    .getResponseCode());

    // Values within the column type bounds.
    HcatTestUtils.createTestDataFile(writeDataFile.getAbsolutePath(),
            new String[] { String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE),
                    String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE) });
    smallTinyIntBoundsCheckHelper(writeDataFile.getAbsolutePath(), ExecJob.JOB_STATUS.COMPLETED);

    // Values outside the column type bounds will fail at runtime.
    HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv",
            new String[] { String.format("%d\t%d", Short.MIN_VALUE - 1, 0) });
    smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/shortTooSmall.tsv", ExecJob.JOB_STATUS.FAILED);

    HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooBig.tsv",
            new String[] { String.format("%d\t%d", Short.MAX_VALUE + 1, 0) });
    smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/shortTooBig.tsv", ExecJob.JOB_STATUS.FAILED);

    HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/byteTooSmall.tsv",
            new String[] { String.format("%d\t%d", 0, Byte.MIN_VALUE - 1) });
    smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/byteTooSmall.tsv", ExecJob.JOB_STATUS.FAILED);

    HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/byteTooBig.tsv",
            new String[] { String.format("%d\t%d", 0, Byte.MAX_VALUE + 1) });
    smallTinyIntBoundsCheckHelper(TEST_DATA_DIR + "/byteTooBig.tsv", ExecJob.JOB_STATUS.FAILED);
}

From source file:org.apache.hcatalog.pig.TestHCatStorerMulti.java

License:Apache License

private void cleanup() throws IOException, CommandNeedRetryException {
    File f = new File(TEST_WAREHOUSE_DIR);
    if (f.exists()) {
        FileUtil.fullyDelete(f);
    }/*w  w  w . j a  va2 s .  c  o m*/
    new File(TEST_WAREHOUSE_DIR).mkdirs();

    dropTable(BASIC_TABLE);
    dropTable(PARTITIONED_TABLE);
}

From source file:org.apache.hive.hcatalog.mapreduce.TestHCatMultiOutputFormat.java

License:Apache License

@BeforeClass
public static void setup() throws Exception {
    System.clearProperty("mapred.job.tracker");
    String testDir = System.getProperty("test.tmp.dir", "./");
    testDir = testDir + "/test_multitable_" + Math.abs(new Random().nextLong()) + "/";
    workDir = new File(new File(testDir).getCanonicalPath());
    FileUtil.fullyDelete(workDir);
    workDir.mkdirs();//from   w  w w  .  ja  v a  2s  .  c  om

    warehousedir = new Path(System.getProperty("test.warehouse.dir"));

    HiveConf metastoreConf = new HiveConf();
    metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString());

    // Run hive metastore server
    msPort = MetaStoreUtils.startMetaStore(metastoreConf);
    // LocalJobRunner does not work with mapreduce OutputCommitter. So need
    // to use MiniMRCluster. MAPREDUCE-2350
    Configuration conf = new Configuration(true);
    conf.set("yarn.scheduler.capacity.root.queues", "default");
    conf.set("yarn.scheduler.capacity.root.default.capacity", "100");

    FileSystem fs = FileSystem.get(conf);
    System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, new JobConf(conf));
    mrConf = mrCluster.createJobConf();

    initializeSetup();

    warehousedir.getFileSystem(conf).mkdirs(warehousedir);
}

From source file:org.apache.hive.hcatalog.mapreduce.TestHCatMultiOutputFormat.java

License:Apache License

@AfterClass
public static void tearDown() throws IOException {
    FileUtil.fullyDelete(workDir);
    FileSystem fs = warehousedir.getFileSystem(hiveConf);
    if (fs.exists(warehousedir)) {
        fs.delete(warehousedir, true);/*  w  w  w.j a va  2  s.c  o  m*/
    }
    if (mrCluster != null) {
        mrCluster.shutdown();
    }
}

From source file:org.apache.hive.hcatalog.mapreduce.TestHCatPartitionPublish.java

License:Apache License

public static File handleWorkDir() throws IOException {
    String testDir = System.getProperty("test.data.dir", "./");
    testDir = testDir + "/test_hcat_partitionpublish_" + Math.abs(new Random().nextLong()) + "/";
    File workDir = new File(new File(testDir).getCanonicalPath());
    FileUtil.fullyDelete(workDir);
    workDir.mkdirs();//from  w  w  w .  ja  v a  2s  .  c  om
    return workDir;
}

From source file:org.apache.hive.hcatalog.mapreduce.TestMultiOutputFormat.java

License:Apache License

private static void createWorkDir() throws IOException {
    String testDir = System.getProperty("test.tmp.dir", "./");
    testDir = testDir + "/test_multiout_" + Math.abs(new Random().nextLong()) + "/";
    workDir = new File(new File(testDir).getCanonicalPath());
    FileUtil.fullyDelete(workDir);
    workDir.mkdirs();//w w  w.  ja  va2 s  .  c o m
}

From source file:org.apache.hive.hcatalog.pig.TestE2EScenarios.java

License:Apache License

@Before
public void setUp() throws Exception {
    File f = new File(TEST_WAREHOUSE_DIR);
    if (f.exists()) {
        FileUtil.fullyDelete(f);
    }//from   ww  w  .j a v  a2  s  .  c  o  m
    if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
        throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR);
    }

    HiveConf hiveConf = new HiveConf(this.getClass());
    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
    hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
            "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
    driver = new Driver(hiveConf);
    SessionState.start(new CliSessionState(hiveConf));

}

From source file:org.apache.hive.hcatalog.pig.TestHCatLoader.java

License:Apache License

@Before
public void setup() throws Exception {
    File f = new File(TEST_WAREHOUSE_DIR);
    if (f.exists()) {
        FileUtil.fullyDelete(f);
    }/*w  w w  .  j av a  2 s  . c o  m*/
    if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
        throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR);
    }

    HiveConf hiveConf = new HiveConf(this.getClass());
    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
    hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
    hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
            "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");

    if (Shell.WINDOWS) {
        WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf);
    }

    driver = new Driver(hiveConf);
    SessionState.start(new CliSessionState(hiveConf));

    createTable(BASIC_TABLE, "a int, b string");
    createTable(COMPLEX_TABLE,
            "name string, studentid int, " + "contact struct<phno:string,email:string>, "
                    + "currently_registered_courses array<string>, " + "current_grades map<string,string>, "
                    + "phnos array<struct<phno:string,type:string>>");

    createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
    createTable(SPECIFIC_SIZE_TABLE, "a int, b string");
    createTable(PARTITIONED_DATE_TABLE, "b string", "dt date");
    AllTypesTable.setupAllTypesTable(driver);

    int LOOP_SIZE = 3;
    String[] input = new String[LOOP_SIZE * LOOP_SIZE];
    basicInputData = new HashMap<Integer, Pair<Integer, String>>();
    int k = 0;
    for (int i = 1; i <= LOOP_SIZE; i++) {
        String si = i + "";
        for (int j = 1; j <= LOOP_SIZE; j++) {
            String sj = "S" + j + "S";
            input[k] = si + "\t" + sj;
            basicInputData.put(k, new Pair<Integer, String>(i, sj));
            k++;
        }
    }
    HcatTestUtils.createTestDataFile(BASIC_FILE_NAME, input);
    HcatTestUtils.createTestDataFile(COMPLEX_FILE_NAME, new String[] {
            "Henry Jekyll\t42\t(415-253-6367,hjekyll@contemporary.edu.uk)\t{(PHARMACOLOGY),(PSYCHIATRY)}\t[PHARMACOLOGY#A-,PSYCHIATRY#B+]\t{(415-253-6367,cell),(408-253-6367,landline)}",
            "Edward Hyde\t1337\t(415-253-6367,anonymous@b44chan.org)\t{(CREATIVE_WRITING),(COPYRIGHT_LAW)}\t[CREATIVE_WRITING#A+,COPYRIGHT_LAW#D]\t{(415-253-6367,cell),(408-253-6367,landline)}", });
    HcatTestUtils.createTestDataFile(DATE_FILE_NAME,
            new String[] { "2016-07-14 08:10:15\tHenry Jekyll", "2016-07-15 11:54:55\tEdward Hyde", });
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    int i = 0;
    server.registerQuery("A = load '" + BASIC_FILE_NAME + "' as (a:int, b:chararray);", ++i);

    server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();",
            ++i);
    server.registerQuery(
            "store A into '" + SPECIFIC_SIZE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
    server.registerQuery("B = foreach A generate a,b;", ++i);
    server.registerQuery("B2 = filter B by a < 2;", ++i);
    server.registerQuery(
            "store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');",
            ++i);

    server.registerQuery("C = foreach A generate a,b;", ++i);
    server.registerQuery("C2 = filter C by a >= 2;", ++i);
    server.registerQuery(
            "store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');",
            ++i);

    server.registerQuery("D = load '" + COMPLEX_FILE_NAME
            + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});",
            ++i);
    server.registerQuery(
            "store D into '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);

    server.registerQuery("E = load '" + DATE_FILE_NAME + "' as (dt:chararray, b:chararray);", ++i);
    server.registerQuery("F = foreach E generate ToDate(dt, 'yyyy-MM-dd HH:mm:ss') as dt, b;", ++i);
    server.registerQuery(
            "store F into '" + PARTITIONED_DATE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();",
            ++i);

    server.executeBatch();
}

From source file:org.apache.hive.hcatalog.pig.TestHCatLoaderEncryption.java

License:Apache License

@Before
public void setup() throws Exception {
    File f = new File(TEST_WAREHOUSE_DIR);
    if (f.exists()) {
        FileUtil.fullyDelete(f);
    }//from w w w. jav a  2 s  . c  o  m
    if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
        throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR);
    }

    HiveConf hiveConf = new HiveConf(this.getClass());
    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
    hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
            "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");

    String s = hiveConf.get("hdfs.minidfs.basedir");
    if (s == null || s.length() <= 0) {
        //return System.getProperty("test.build.data", "build/test/data") + "/dfs/";
        hiveConf.set("hdfs.minidfs.basedir", System.getProperty("test.build.data", "build/test/data") + "_"
                + System.currentTimeMillis() + "_" + salt.getAndIncrement() + "/dfs/");
    }
    if (Shell.WINDOWS) {
        WindowsPathUtil.convertPathsFromWindowsToHdfs(hiveConf);
    }

    driver = new Driver(hiveConf);

    initEncryptionShim(hiveConf);
    String encryptedTablePath = TEST_WAREHOUSE_DIR + "/encryptedTable";
    SessionState.start(new CliSessionState(hiveConf));

    SessionState.get().out = System.out;

    createTable(BASIC_TABLE, "a int, b string");
    createTableInSpecifiedPath(ENCRYPTED_TABLE, "a int, b string",
            WindowsPathUtil.getHdfsUriString(encryptedTablePath), driver);

    associateEncryptionZoneWithPath(WindowsPathUtil.getHdfsUriString(encryptedTablePath));

    int LOOP_SIZE = 3;
    String[] input = new String[LOOP_SIZE * LOOP_SIZE];
    basicInputData = new HashMap<Integer, Pair<Integer, String>>();
    int k = 0;
    for (int i = 1; i <= LOOP_SIZE; i++) {
        String si = i + "";
        for (int j = 1; j <= LOOP_SIZE; j++) {
            String sj = "S" + j + "S";
            input[k] = si + "\t" + sj;
            basicInputData.put(k, new Pair<Integer, String>(i, sj));
            k++;
        }
    }
    HcatTestUtils.createTestDataFile(BASIC_FILE_NAME, input);
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    int i = 0;
    server.registerQuery("A = load '" + BASIC_FILE_NAME + "' as (a:int, b:chararray);", ++i);
    server.registerQuery(
            "store A into '" + ENCRYPTED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
    server.executeBatch();
}

From source file:org.apache.hive.hcatalog.pig.TestHCatLoaderPredicatePushDown.java

License:Apache License

private static void setUpLocalFileSystemDirectories() {
    File f = new File(TEST_DATA_DIR);
    if (f.exists()) {
        FileUtil.fullyDelete(f);
    }/*  w ww.  j av a 2s.  co m*/
    if (!(new File(TEST_DATA_DIR).mkdirs())) {
        throw new RuntimeException(
                "Could not create test-directory " + TEST_DATA_DIR + " on local filesystem.");
    }
}