Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static void deleteFileIfExistOnLFS(Path outpath, JobConf job) throws IOException {
    if (FileSystem.getLocal(job).exists(outpath)) {
        FileSystem.getLocal(job).delete(outpath, true);
    }//from www.  j  a v  a2  s  . c  o m
}

From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java

License:Open Source License

/**
 *    //from   w w w  . jav a  2  s . co  m
 * @param appId
 * @throws ParseException
 * @throws IOException
 * @throws DMLRuntimeException
 * @throws InterruptedException 
 */
@SuppressWarnings("deprecation")
private void copyResourcesToHdfsWorkingDir(YarnConfiguration yconf, String hdfsWD)
        throws ParseException, IOException, DMLRuntimeException, InterruptedException {
    FileSystem fs = FileSystem.get(yconf);

    //create working directory
    MapReduceTool.createDirIfNotExistOnHDFS(hdfsWD, DMLConfig.DEFAULT_SHARED_DIR_PERMISSION);

    //serialize the dml config to HDFS file 
    //NOTE: we do not modify and ship the absolute scratch space path of the current user
    //because this might result in permission issues if the app master is run with a different user
    //(runtime plan migration during resource reoptimizations now needs to use qualified names
    //for shipping/reading intermediates) TODO modify resource reoptimizer on prototype integration.
    Path confPath = new Path(hdfsWD, DML_CONFIG_NAME);
    FSDataOutputStream fout = fs.create(confPath, true);
    //_dmlConfig.makeQualifiedScratchSpacePath(); 
    fout.writeBytes(_dmlConfig.serializeDMLConfig() + "\n");
    fout.close();
    _hdfsDMLConfig = confPath.makeQualified(fs).toString();
    LOG.debug("DML config written to HDFS file: " + _hdfsDMLConfig + "");

    //serialize the dml script to HDFS file
    Path scriptPath = new Path(hdfsWD, DML_SCRIPT_NAME);
    FSDataOutputStream fout2 = fs.create(scriptPath, true);
    fout2.writeBytes(_dmlScript);
    fout2.close();
    _hdfsDMLScript = scriptPath.makeQualified(fs).toString();
    LOG.debug("DML script written to HDFS file: " + _hdfsDMLScript + "");

    // copy local jar file to HDFS (try to get the original jar filename)
    String fname = getLocalJarFileNameFromEnvConst();
    if (fname == null) {
        //get location of unpacked jar classes and repackage (if required)
        String lclassFile = DMLYarnClient.class.getProtectionDomain().getCodeSource().getLocation().getPath()
                .toString();
        File flclassFile = new File(lclassFile);
        if (!flclassFile.isDirectory()) //called w/ jar 
            fname = lclassFile;
        else //called w/ unpacked jar (need to be repackaged)   
            fname = createJar(lclassFile);
    }
    Path srcPath = new Path(fname);
    Path dstPath = new Path(hdfsWD, srcPath.getName());
    FileUtil.copy(FileSystem.getLocal(yconf), srcPath, fs, dstPath, false, true, yconf);
    _hdfsJarFile = dstPath.makeQualified(fs).toString();
    LOG.debug(
            "Jar file copied from local file: " + srcPath.toString() + " to HDFS file: " + dstPath.toString());
}

From source file:com.ibm.jaql.JaqlScriptTestCase.java

License:Apache License

protected void runScript(Mode mode) throws Exception {
    String testLabel = script + "." + mode;
    try {//from w  ww. j a v a  2 s .c  o  m
        String runMode = System.getProperty("test." + mode, "true");
        if (!runMode.equals("true")) {
            System.err.println("\nSkipping disabled jaql test " + testLabel + " (test." + mode + "=" + runMode
                    + " != true)\n");
            return;
        }

        String jaqlHome = System.getProperty("jaql.home", ".");
        jaqlHome = new File(jaqlHome).getAbsolutePath().toString().replace('\\', '/') + "/";

        String scriptDir = jaqlHome + getScriptDir();
        String[] moduleDirs = getModuleDirs();
        String queriesName = scriptDir + script + "Queries.txt";
        String goldName = scriptDir + testLabel + ".gold";

        if (!new File(goldName).exists()) {
            // look for the mode-independent gold file
            if (mode == Mode.COUNT) {
                System.err.println("\nSkipping jaql count test " + testLabel + " (no gold file)\n");
                return;
            }
            goldName = scriptDir + script + ".gold";
            if (!new File(goldName).exists()) {
                Assert.fail("\nNo gold file for jaql test " + testLabel + "at path: " + goldName);
                return;
            }
        }

        System.err.println("\nRunning jaql test " + testLabel + "\n");

        String outDir = jaqlHome + "build/test/";
        String workDir = outDir + "run." + testLabel + "/";
        String outName = workDir + testLabel + ".out";
        new File(workDir).mkdirs();

        // Set the default directories
        System.setProperty("jaql.local.dir", workDir);
        Configuration conf = new Configuration();
        LocalFileSystem lfs = FileSystem.getLocal(conf);
        lfs.setWorkingDirectory(new Path(workDir));
        FileSystem fs = FileSystem.get(conf);
        if (!(fs instanceof LocalFileSystem)) {
            String user = UnixUserGroupInformation.login(conf).getUserName();
            fs.setWorkingDirectory(new Path("/temp/" + user + "/com.ibm.jaql/test/" + script));
        }
        // mapred.working.dir is automatically set from the fs, but only once. 
        // When running multiple tests in the same JVM, it only picks up the first setting.
        if (Globals.getJobConf() != null) {
            Globals.getJobConf().setWorkingDirectory(fs.getWorkingDirectory());
        }

        // make tests work the same on windows as unix.
        System.setProperty("line.separator", "\n");
        final FastPrintWriter resultStream = new FastPrintWriter(new FileWriter(outName));
        Reader queryReader = new InputStreamReader(new FileInputStream(queriesName), "UTF-8");
        queryReader = new EchoedReader(queryReader, new FastPrintStream(System.err));
        queryReader = new EchoedReader(queryReader, resultStream);

        ClassLoaderMgr.reset();

        Jaql jaql = new Jaql(queriesName, queryReader);
        jaql.setModulePath(moduleDirs);

        if (mode == Mode.COUNT) {
            final Class<?>[] exprsToCount = new Class<?>[] { AbstractReadExpr.class, AbstractWriteExpr.class,
                    MapReduceBaseExpr.class, };
            jaql.setExplainHandler(new CountExplainHandler(resultStream, exprsToCount));
            jaql.setExplainOnly(true);
        } else if (mode == Mode.DECOMPILE) {
            jaql.setExplainHandler(new DecompileExplainHandler(System.err));
            jaql.setExplainOnly(true);
        }

        jaql.setExceptionHandler(new TestExceptionHandler(resultStream, jaqlHome));
        jaql.enableRewrite(mode != Mode.NO_REWRITE);
        boolean schemaPrinting = "schemaPrinting".equals(script);
        jaql.setJaqlPrinter(new TestPrinter(resultStream, schemaPrinting));

        String extJar = getExtensionJar();
        if (extJar != null)
            jaql.addJar(jaqlHome + extJar);
        jaql.setVar(DATADIR_NAME, DATADIR_VALUE);

        // run the script
        jaql.run();

        // finish up
        jaql.close();
        queryReader.close();
        resultStream.close();

        // compare with expected output
        boolean diff = compareResults(outName, goldName);
        if (diff) {
            String msg = "Found differences during jaql test " + testLabel;
            System.err.println("\n" + msg);
            Assert.fail(msg);
        }

        System.err.println("\nSuccessfully ran jaql test " + testLabel + "\n");
    } catch (Exception e) {
        e.printStackTrace(System.err);
        System.err.println("\n\nFailure of jaql test " + testLabel);
        Assert.fail(e.getMessage());
    }
}

From source file:com.inmobi.conduit.CompressedFileReaderTest.java

License:Apache License

private void uncompress(String fileName) throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs;//from  w ww.j  ava2  s . co  m
    fs = FileSystem.getLocal(conf);

    CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf);
    CompressionCodec codec = codecFactory.getCodec(new Path(fileName));
    if (codec == null) {
        System.out.println("cant find codec");
        System.exit(1);
    }
    LOG.info("Using compression codec [" + codec.toString() + "]");
    CompressionInputStream is = codec.createInputStream(fs.open(new Path(fileName)));
    OutputStream out = null;
    try {
        String outputURI = CompressionCodecFactory.removeSuffix(fileName, codec.getDefaultExtension());
        out = fs.create(new Path(outputURI + "-uncompressed"));
        org.apache.hadoop.io.IOUtils.copyBytes(is, out, conf);
    } finally {
        org.apache.hadoop.io.IOUtils.closeStream(out);
        IOUtils.closeStream(is);

    }
}

From source file:com.inmobi.conduit.distcp.tools.DistCp.java

License:Apache License

/**
 * Add SSL files to distributed cache. Trust store, key store and ssl config xml
 *
 * @param configuration - Job configuration
 * @param sslConfigPath - ssl Configuration file specified through options
 * @throws IOException - If any//from  www.java  2 s  .  co m
 */
private void addSSLFilesToDistCache(Configuration configuration, Path sslConfigPath) throws IOException {
    FileSystem localFS = FileSystem.getLocal(configuration);

    Configuration sslConf = new Configuration(false);
    sslConf.addResource(sslConfigPath);

    Path localStorePath = getLocalStorePath(sslConf, "ssl.client.truststore.location");
    DistributedCache.addCacheFile(localStorePath.makeQualified(localFS).toUri(), configuration);
    configuration.set("ssl.client.truststore.location", localStorePath.getName());

    localStorePath = getLocalStorePath(sslConf, "ssl.client.keystore.location");
    DistributedCache.addCacheFile(localStorePath.makeQualified(localFS).toUri(), configuration);
    configuration.set("ssl.client.keystore.location", localStorePath.getName());

    DistributedCache.addCacheFile(sslConfigPath.makeQualified(localFS).toUri(), configuration);
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void testPurgerService(String testfilename, int numofhourstoadd, boolean checkifexists,
        boolean checktrashexists) throws Exception {
    ConduitConfigParser configparser = new ConduitConfigParser(testfilename);
    ConduitConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {
        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar todaysdate = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        todaysdate.add(Calendar.HOUR, numofhourstoadd);

        createTestPurgefiles(fs, cluster, todaysdate, false);

        service.runOnce();//ww w  . j  a v a  2  s.c  om

        verifyPurgefiles(fs, cluster, todaysdate, checkifexists, checktrashexists);
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

public void testDataPurgerPartition() throws Exception {
    LOG.info("Check data purger does not stop when unable to delete a path");
    ConduitConfigParser configparser = new ConduitConfigParser("test-dps-conduit_X_hcat_5.xml");
    ConduitConfig config = configparser.getConfig();
    Conduit.setHCatEnabled(true);//w w w.j a v  a2 s  . c  o m
    Conduit.setHcatDBName("conduit");

    for (Cluster cluster : config.getClusters().values()) {
        HiveConf hcatConf1 = TestHCatUtil.getHcatConf(20109, "target/metaStore-purger1", "metadb-purger1");
        TestHCatUtil.startMetaStoreServer(hcatConf1, 20109);
        Thread.sleep(10000);

        Conduit.setHiveConf(hcatConf1);

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        TestHCatUtil.createDatabase(Conduit.getHcatDBName());
        TestHCatUtil thutil = new TestHCatUtil();
        Table table = null;
        for (String stream : cluster.getSourceStreams()) {
            table = thutil.createTable(Conduit.getHcatDBName(), "conduit_local_" + stream);
        }
        Calendar date1 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date1.add(Calendar.HOUR, -7);
        createTestPurgePartitionFiles(fs, cluster, date1, table);

        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        service.runOnce();

        verifyPurgePartitionFiles(fs, cluster, date1, false, false, table);
        service.clearStreamHCatEnableMap();
        Conduit.setHCatEnabled(false);
        TestHCatUtil.stop();
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

public void testDataPurger() throws Exception {
    AbstractService.clearHCatInMemoryMaps();
    LOG.info("Check data purger does not stop when unable to delete a path");
    ConduitConfigParser configparser = new ConduitConfigParser("test-dps-conduit_X_5.xml");
    ConduitConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar date1 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date1.add(Calendar.HOUR, -7);
        createTestPurgefiles(fs, cluster, date1, false);
        Calendar date2 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date2.add(Calendar.HOUR, -6);
        createTestPurgefiles(fs, cluster, date2, false);
        ArrayList<Path> pathsToProcess = new ArrayList<Path>();
        Path[] paths = getLocalCommitPath(fs, cluster, date2);
        for (Path path : paths) {
            fs.setPermission(path, new FsPermission("000"));
            pathsToProcess.add(path);/*from  w w  w  .  j  ava2s . c  o m*/
        }
        paths = getMergeCommitPath(fs, cluster, date2);
        for (Path path : paths) {
            fs.setPermission(path, new FsPermission("000"));
            pathsToProcess.add(path);
        }
        Calendar date3 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date3.add(Calendar.HOUR, -5);
        createTestPurgefiles(fs, cluster, date3, false);

        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        service.runOnce();

        verifyPurgefiles(fs, cluster, date1, false, false);
        verifyPurgefiles(fs, cluster, date2, true, false);
        verifyPurgefiles(fs, cluster, date3, false, false);
        for (Path p : pathsToProcess) {
            fs.setPermission(p, new FsPermission("755"));
        }
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }

    Assert.assertEquals(ConduitMetrics.<SlidingTimeWindowGauge>getMetric("DataPurgerService",
            "purgePaths.count", DataPurgerService.class.getName()).getValue().longValue(), 9);
    Assert.assertEquals(ConduitMetrics.<SlidingTimeWindowGauge>getMetric("DataPurgerService",
            "deleteFailures.count", DataPurgerService.class.getName()).getValue().longValue(), 0);
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

public void testTrashPurging() throws Exception {
    LOG.info("Creating empty data dirs");
    ConduitConfigParser configparser = new ConduitConfigParser("test-dps-conduit_X_6.xml");
    ConduitConfig config = configparser.getConfig();

    for (Cluster cluster : config.getClusters().values()) {

        FileSystem fs = FileSystem.getLocal(new Configuration());
        fs.delete(new Path(cluster.getRootDir()), true);

        Calendar date1 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date1.add(Calendar.HOUR, -48);
        createTestPurgefiles(fs, cluster, date1, true);
        Calendar date2 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date2.add(Calendar.HOUR, -24);
        createTestPurgefiles(fs, cluster, date2, true);
        Calendar date3 = new GregorianCalendar(Calendar.getInstance().getTimeZone());
        date3.add(Calendar.HOUR, -1);
        createTestPurgefiles(fs, cluster, date3, false);

        TestDataPurgerService service = new TestDataPurgerService(config, cluster);

        service.runOnce();/*from w  ww .  j a  v  a2 s.c o m*/

        verifyPurgefiles(fs, cluster, date1, false, false);
        verifyPurgefiles(fs, cluster, date2, false, false);
        verifyPurgefiles(fs, cluster, date3, true, true);
        fs.delete(new Path(cluster.getRootDir()), true);
        fs.close();
    }

    Assert.assertEquals(ConduitMetrics.<SlidingTimeWindowGauge>getMetric("DataPurgerService",
            "purgePaths.count", DataPurgerService.class.getName()).getValue().longValue(), 6);
}

From source file:com.inmobi.databus.files.TestHadoopStreamFile.java

License:Apache License

@Test
public void testHadoopStreamFile() throws IOException {
    rootDir = new Path(TestUtil.getConfiguredRootDir(), this.getClass().getSimpleName());
    fs = FileSystem.getLocal(new Configuration());
    Path p1 = new Path(rootDir, "2012/12/12/12/11");
    Path p2 = new Path(rootDir, "2012/12/12/12/12");
    Long t1 = 1L;/*from   w w  w.j  a  va 2s.  c  o  m*/
    Long t2 = 2L;
    String f1 = "f1";
    String f2 = "f2";
    HadoopStreamFile hp1 = new HadoopStreamFile(p1, null, null);
    HadoopStreamFile hp2 = new HadoopStreamFile(p2, null, null);
    HadoopStreamFile h11 = new HadoopStreamFile(p1, f1, t1);
    HadoopStreamFile h12 = new HadoopStreamFile(p1, f2, t2);
    HadoopStreamFile h21 = new HadoopStreamFile(p2, f1, t1);
    HadoopStreamFile h22 = new HadoopStreamFile(p2, f2, t2);

    Assert.assertTrue(hp1.equals(h11));
    Assert.assertTrue(hp1.equals(h12));
    Assert.assertTrue(hp2.equals(h21));
    Assert.assertTrue(hp2.equals(h22));
    Assert.assertTrue(hp1.compareTo(h21) < 0);
    Assert.assertTrue(hp1.compareTo(h21) < 0);
    Assert.assertTrue(h11.compareTo(h12) < 0);
    Assert.assertTrue(h12.compareTo(h21) < 0);
    Assert.assertTrue(h21.compareTo(h22) < 0);

    fs.mkdirs(p1);
    Path pf11 = new Path(p1, f1);
    fs.create(pf11);
    FileStatus fs11 = fs.getFileStatus(pf11);
    Assert.assertEquals(HadoopStreamFile.create(fs11).toString(), new Path("2012/12/12/12/11/f1").toString());

    fs.delete(p1, true);
}