Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

public void testFailedImportDueToIOException() throws IOException {
    // Make sure that if a MapReduce job to do the import fails due
    // to an IOException, we tell the user about it.

    // Create a table to attempt to import.
    createTableForColType("VARCHAR(32)", "'meep'");

    Configuration conf = new Configuration();

    LogFactory.getLog(getClass()).info(" getWarehouseDir() " + getWarehouseDir());

    // Make the output dir exist so we know the job will fail via IOException.
    Path outputPath = new Path(new Path(getWarehouseDir()), getTableName());
    FileSystem fs = FileSystem.getLocal(conf);
    fs.mkdirs(outputPath);//from  www .  ja  va  2s .  c  o  m

    assertTrue(fs.exists(outputPath));

    String[] argv = getArgv(true, new String[] { "DATA_COL0" }, conf);

    Sqoop importer = new Sqoop(new ImportTool());
    try {
        int ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected ImportException running this job.", 1 == ret);
    } catch (Exception e) {
        // In debug mode, IOException is wrapped in RuntimeException.
        LOG.info("Got exceptional return (expected: ok). msg is: " + e);
    }
}

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

public void testFailedNoColumns() throws IOException {
    // Make sure that if a MapReduce job to do the import fails due
    // to an IOException, we tell the user about it.

    // Create a table to attempt to import.
    createTableForColType("VARCHAR(32)", "'meep'");

    Configuration conf = new Configuration();

    // Make the output dir exist so we know the job will fail via IOException.
    Path outputPath = new Path(new Path(getWarehouseDir()), getTableName());
    FileSystem fs = FileSystem.getLocal(conf);
    fs.mkdirs(outputPath);//from w  w w. j av  a  2s . c o  m
    assertTrue(fs.exists(outputPath));

    String[] argv = getArgv(true, new String[] { "" }, conf);

    Sqoop importer = new Sqoop(new ImportTool());
    try {
        int ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected job to fail due to no colnames.", 1 == ret);
    } catch (Exception e) {
        // In debug mode, IOException is wrapped in RuntimeException.
        LOG.info("Got exceptional return (expected: ok). msg is: " + e);
    }
}

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

public void testFailedIllegalColumns() throws IOException {
    // Make sure that if a MapReduce job to do the import fails due
    // to an IOException, we tell the user about it.

    // Create a table to attempt to import.
    createTableForColType("VARCHAR(32)", "'meep'");

    Configuration conf = new Configuration();

    // Make the output dir exist so we know the job will fail via IOException.
    Path outputPath = new Path(new Path(getWarehouseDir()), getTableName());
    FileSystem fs = FileSystem.getLocal(conf);
    fs.mkdirs(outputPath);/*w  w w.j  a  v a2s . co  m*/

    assertTrue(fs.exists(outputPath));

    // DATA_COL0 ok, by zyzzyva not good
    String[] argv = getArgv(true, new String[] { "DATA_COL0", "zyzzyva" }, conf);

    Sqoop importer = new Sqoop(new ImportTool());
    try {
        int ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected job to fail due bad colname.", 1 == ret);
    } catch (Exception e) {
        // In debug mode, IOException is wrapped in RuntimeException.
        LOG.info("Got exceptional return (expected: ok). msg is: " + e);
    }
}

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

public void testDuplicateColumns() throws IOException {
    // Make sure that if a MapReduce job to do the import fails due
    // to an IOException, we tell the user about it.

    // Create a table to attempt to import.
    createTableForColType("VARCHAR(32)", "'meep'");

    Configuration conf = new Configuration();

    // Make the output dir exist so we know the job will fail via IOException.
    Path outputPath = new Path(new Path(getWarehouseDir()), getTableName());
    FileSystem fs = FileSystem.getLocal(conf);
    fs.mkdirs(outputPath);//from  w  w w. j  av a  2s.  c  o m
    assertTrue(fs.exists(outputPath));

    String[] argv = getArgv(true, new String[] { "DATA_COL0,DATA_COL0" }, conf);

    Sqoop importer = new Sqoop(new ImportTool());
    try {
        int ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected job to fail!", 1 == ret);
    } catch (Exception e) {
        // In debug mode, ImportException is wrapped in RuntimeException.
        LOG.info("Got exceptional return (expected: ok). msg is: " + e);
    }
}

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

private String[] getContent(Configuration conf, Path path) throws Exception {
    FileSystem fs = FileSystem.getLocal(conf);
    FileStatus[] stats = fs.listStatus(path);
    String[] fileNames = new String[stats.length];
    for (int i = 0; i < stats.length; i++) {
        fileNames[i] = stats[i].getPath().toString();
    }/*w  w w.j  av a2s . co m*/

    // Read all the files adding the value lines to the list.
    List<String> strings = new ArrayList<String>();
    for (String fileName : fileNames) {
        if (fileName.startsWith("_") || fileName.startsWith(".")) {
            continue;
        }

        SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
        WritableComparable key = (WritableComparable) reader.getKeyClass().newInstance();
        Writable value = (Writable) reader.getValueClass().newInstance();
        while (reader.next(key, value)) {
            strings.add(value.toString());
        }
    }
    return strings.toArray(new String[0]);
}

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

public void testDeleteTargetDir() throws Exception {
    // Make sure that if a MapReduce job to do the import fails due
    // to an IOException, we tell the user about it.

    // Create a table to attempt to import.
    createTableForColType("VARCHAR(32)", "'meep'");

    Configuration conf = new Configuration();

    // Make the output dir does not exist
    Path outputPath = new Path(new Path(getWarehouseDir()), getTableName());
    FileSystem fs = FileSystem.getLocal(conf);
    fs.delete(outputPath, true);/*from ww w  .  j a  v  a2  s  . c  om*/
    assertTrue(!fs.exists(outputPath));

    String[] argv = getArgv(true, new String[] { "DATA_COL0" }, conf);
    argv = Arrays.copyOf(argv, argv.length + 1);
    argv[argv.length - 1] = "--delete-target-dir";

    Sqoop importer = new Sqoop(new ImportTool());
    try {
        int ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected job to go through if target directory" + " does not exist.", 0 == ret);
        assertTrue(fs.exists(outputPath));
        // expecting one _SUCCESS file and one file containing data
        assertTrue("Expecting two files in the directory.", fs.listStatus(outputPath).length == 2);
        String[] output = getContent(conf, outputPath);
        assertEquals("Expected output and actual output should be same.", "meep", output[0]);

        ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected job to go through if target directory exists.", 0 == ret);
        assertTrue(fs.exists(outputPath));
        // expecting one _SUCCESS file and one file containing data
        assertTrue("Expecting two files in the directory.", fs.listStatus(outputPath).length == 2);
        output = getContent(conf, outputPath);
        assertEquals("Expected output and actual output should be same.", "meep", output[0]);
    } catch (Exception e) {
        // In debug mode, ImportException is wrapped in RuntimeException.
        LOG.info("Got exceptional return (expected: ok). msg is: " + e);
    }
}

From source file:com.cloudera.sqoop.TestAppendUtils.java

License:Apache License

/**
 * If the append source does not exist, don't crash.
 *///w  w  w.  j  a  va  2 s .c  o m
public void testAppendSrcDoesNotExist() throws IOException {
    Configuration conf = new Configuration();
    if (!isOnPhysicalCluster()) {
        conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    SqoopOptions options = new SqoopOptions(conf);
    options.setTableName("meep");
    Path missingPath = new Path("doesNotExistForAnyReason");
    FileSystem local = FileSystem.getLocal(conf);
    assertFalse(local.exists(missingPath));
    ImportJobContext importContext = new ImportJobContext("meep", null, options, missingPath);
    AppendUtils utils = new AppendUtils(importContext);
    utils.append();
}

From source file:com.cloudera.sqoop.TestIncrementalImport.java

License:Apache License

/**
 * Delete all files in a directory for a table.
 *///from   www .jav  a  2s  .  c o m
public void clearDir(String tableName) {
    try {
        FileSystem fs = FileSystem.getLocal(new Configuration());
        Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
        Path tableDir = new Path(warehouse, tableName);
        fs.delete(tableDir, true);
    } catch (Exception e) {
        fail("Got unexpected exception: " + StringUtils.stringifyException(e));
    }
}

From source file:com.cloudera.sqoop.TestIncrementalImport.java

License:Apache License

/**
 * Look at a directory that should contain files full of an imported 'id'
 * column. Assert that all numbers in [0, expectedNums) are present
 * in order./*from  www  .  j av  a 2  s . com*/
 */
public void assertDirOfNumbers(String tableName, int expectedNums) {
    try {
        FileSystem fs = FileSystem.getLocal(new Configuration());
        Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
        Path tableDir = new Path(warehouse, tableName);
        FileStatus[] stats = fs.listStatus(tableDir);
        String[] fileNames = new String[stats.length];
        for (int i = 0; i < stats.length; i++) {
            fileNames[i] = stats[i].getPath().toString();
        }

        Arrays.sort(fileNames);

        // Read all the files in sorted order, adding the value lines to the list.
        List<String> receivedNums = new ArrayList<String>();
        for (String fileName : fileNames) {
            if (fileName.startsWith("_") || fileName.startsWith(".")) {
                continue;
            }

            BufferedReader r = new BufferedReader(new InputStreamReader(fs.open(new Path(fileName))));
            try {
                while (true) {
                    String s = r.readLine();
                    if (null == s) {
                        break;
                    }

                    receivedNums.add(s.trim());
                }
            } finally {
                r.close();
            }
        }

        assertEquals(expectedNums, receivedNums.size());

        // Compare the received values with the expected set.
        for (int i = 0; i < expectedNums; i++) {
            assertEquals((int) i, (int) Integer.valueOf(receivedNums.get(i)));
        }
    } catch (Exception e) {
        fail("Got unexpected exception: " + StringUtils.stringifyException(e));
    }
}

From source file:com.cloudera.sqoop.TestIncrementalImport.java

License:Apache License

/**
 * Assert that a directory contains a file with exactly one line
 * in it, containing the prescribed number 'val'.
 *///from  ww  w .j  a  v  a 2 s .  c  om
public void assertSpecificNumber(String tableName, int val) {
    try {
        FileSystem fs = FileSystem.getLocal(new Configuration());
        Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
        Path tableDir = new Path(warehouse, tableName);
        FileStatus[] stats = fs.listStatus(tableDir);
        String[] filePaths = new String[stats.length];
        for (int i = 0; i < stats.length; i++) {
            filePaths[i] = stats[i].getPath().toString();
        }

        // Read the first file that is not a hidden file.
        boolean foundVal = false;
        for (String filePath : filePaths) {
            String fileName = new Path(filePath).getName();
            if (fileName.startsWith("_") || fileName.startsWith(".")) {
                continue;
            }

            if (foundVal) {
                // Make sure we don't have two or more "real" files in the dir.
                fail("Got an extra data-containing file in this directory.");
            }

            BufferedReader r = new BufferedReader(new InputStreamReader(fs.open(new Path(filePath))));
            try {
                String s = r.readLine();
                if (null == s) {
                    fail("Unexpected empty file " + filePath + ".");
                }
                assertEquals(val, (int) Integer.valueOf(s.trim()));

                String nextLine = r.readLine();
                if (nextLine != null) {
                    fail("Expected only one result, but got another line: " + nextLine);
                }

                // Successfully got the value we were looking for.
                foundVal = true;
            } finally {
                r.close();
            }
        }
    } catch (IOException e) {
        fail("Got unexpected exception: " + StringUtils.stringifyException(e));
    }
}