Example usage for org.apache.hadoop.fs Path SEPARATOR

List of usage examples for org.apache.hadoop.fs Path SEPARATOR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:org.apache.tez.analyzer.plugins.TezAnalyzerBase.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    //Parse downloaded contents
    CommandLine cmdLine = null;//  w  w  w.j a va 2  s .co m
    try {
        cmdLine = new GnuParser().parse(buildOptions(), args);
    } catch (ParseException e) {
        System.err.println("Invalid options on command line");
        printUsage();
        return -1;
    }
    saveResults = cmdLine.hasOption(SAVE_RESULTS);

    if (cmdLine.hasOption(HELP)) {
        printUsage();
        return 0;
    }

    outputDir = cmdLine.getOptionValue(OUTPUT_DIR);
    if (outputDir == null) {
        outputDir = System.getProperty("user.dir");
    }

    File file = null;
    if (cmdLine.hasOption(EVENT_FILE_NAME)) {
        file = new File(cmdLine.getOptionValue(EVENT_FILE_NAME));
    }

    String dagId = cmdLine.getOptionValue(DAG_ID);

    DagInfo dagInfo = null;

    if (file == null) {
        if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
            System.err.println("Event file name must be specified when using simple history");
            printUsage();
            return -2;
        }
        // using ATS - try to download directly
        String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir };

        int result = ATSImportTool.process(importArgs);
        if (result != 0) {
            System.err.println("Error downloading data from ATS");
            return -3;
        }

        //Parse ATS data and verify results
        //Parse downloaded contents
        file = new File(outputDir + Path.SEPARATOR + dagId + Path.SEPARATOR + dagId + ".zip");
    }

    Preconditions.checkState(file != null);
    if (!cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
        ATSFileParser parser = new ATSFileParser(file);
        dagInfo = parser.getDAGData(dagId);
    } else {
        SimpleHistoryParser parser = new SimpleHistoryParser(file);
        dagInfo = parser.getDAGData(dagId);
    }
    Preconditions.checkState(dagInfo.getDagId().equals(dagId));
    analyze(dagInfo);
    Result result = getResult();
    if (saveResults && (result instanceof CSVResult)) {
        String fileName = outputDir + File.separator + this.getClass().getName() + "_" + dagInfo.getDagId()
                + ".csv";
        ((CSVResult) result).dumpToFile(fileName);
        System.out.println("Saved results in " + fileName);
    }
    return 0;
}

From source file:org.apache.tez.analyzer.TestAnalyzer.java

License:Apache License

private DagInfo getDagInfo(String dagId) throws Exception {
    // sleep for a bit to let ATS events be sent from AM
    DagInfo dagInfo = null;/*from  ww w.  j a  va2  s . c o  m*/
    if (usingATS) {
        //Export the data from ATS
        String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR,
                "--yarnTimelineAddress=" + yarnTimelineAddress };

        int result = ATSImportTool.process(args);
        assertTrue(result == 0);

        //Parse ATS data and verify results
        //Parse downloaded contents
        File downloadedFile = new File(DOWNLOAD_DIR + Path.SEPARATOR + dagId + ".zip");
        ATSFileParser parser = new ATSFileParser(downloadedFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    } else {
        if (!downloadedSimpleHistoryFile) {
            downloadedSimpleHistoryFile = true;
            TezDAGID tezDAGID = TezDAGID.fromString(dagId);
            ApplicationAttemptId applicationAttemptId = ApplicationAttemptId
                    .newInstance(tezDAGID.getApplicationId(), 1);
            Path historyPath = new Path(miniTezCluster.getConfig().get("fs.defaultFS") + SIMPLE_HISTORY_DIR
                    + HISTORY_TXT + "." + applicationAttemptId);
            FileSystem fs = historyPath.getFileSystem(miniTezCluster.getConfig());

            Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT);
            fs.copyToLocalFile(historyPath, localPath);
        }
        //Now parse via SimpleHistory
        File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT);
        SimpleHistoryParser parser = new SimpleHistoryParser(localFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    }
    return dagInfo;
}

From source file:org.apache.tez.auxservices.TestShuffleHandler.java

License:Apache License

private static void createShuffleHandlerFiles(File logDir, String user, String appId, String appAttemptId,
        Configuration conf, List<File> fileMap) throws IOException {
    String attemptDir = StringUtils.join(Path.SEPARATOR,
            new String[] { logDir.getAbsolutePath(), ShuffleHandler.USERCACHE, user, ShuffleHandler.APPCACHE,
                    appId, "dag_1/" + "output", appAttemptId });
    File appAttemptDir = new File(attemptDir);
    appAttemptDir.mkdirs();/*from  w w w.ja  va2 s  .c o m*/
    System.out.println(appAttemptDir.getAbsolutePath());
    File indexFile = new File(appAttemptDir, "file.out.index");
    fileMap.add(indexFile);
    createIndexFile(indexFile, conf);
    File mapOutputFile = new File(appAttemptDir, "file.out");
    fileMap.add(mapOutputFile);
    createMapOutputFile(mapOutputFile, conf);
}

From source file:org.apache.tez.auxservices.TestShuffleHandler.java

License:Apache License

@Test(timeout = 5000)
public void testDagDelete() throws Exception {
    final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
    Configuration conf = new Configuration();
    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple");
    UserGroupInformation.setConfiguration(conf);
    File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
    ApplicationId appId = ApplicationId.newInstance(12345, 1);
    String appAttemptId = "attempt_12345_1_m_1_0";
    String user = "randomUser";
    List<File> fileMap = new ArrayList<File>();
    createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap);
    ShuffleHandler shuffleHandler = new ShuffleHandler() {
        @Override//from  w  ww .java 2s .c o  m
        protected Shuffle getShuffle(Configuration conf) {
            // replace the shuffle handler with one stubbed for testing
            return new Shuffle(conf) {
                @Override
                protected void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) {
                    if (failures.size() == 0) {
                        failures.add(new Error(message));
                        ctx.getChannel().close();
                    }
                }
            };
        }
    };
    shuffleHandler.init(conf);
    try {
        shuffleHandler.start();
        DataOutputBuffer outputBuffer = new DataOutputBuffer();
        outputBuffer.reset();
        Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(),
                "password".getBytes(), new Text(user), new Text("shuffleService"));
        jt.write(outputBuffer);
        shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId,
                ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength())));
        URL url = new URL(
                "http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
                        + "/mapOutput?dagAction=delete&job=job_12345_0001&dag=1");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
        conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
        String dagDirStr = StringUtils.join(Path.SEPARATOR, new String[] { absLogDir.getAbsolutePath(),
                ShuffleHandler.USERCACHE, user, ShuffleHandler.APPCACHE, appId.toString(), "dag_1/" });
        File dagDir = new File(dagDirStr);
        Assert.assertTrue("Dag Directory does not exist!", dagDir.exists());
        conn.connect();
        try {
            DataInputStream is = new DataInputStream(conn.getInputStream());
            is.close();
            Assert.assertFalse("Dag Directory was not deleted!", dagDir.exists());
        } catch (EOFException e) {
            // ignore
        }
        Assert.assertEquals("sendError called due to shuffle error", 0, failures.size());
    } finally {
        shuffleHandler.stop();
        FileUtil.fullyDelete(absLogDir);
    }
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testCreateTezSysStagingPath() throws Exception {
    String strAppId = "testAppId";
    String expectedStageDir = RESOLVED_STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR
            + Path.SEPARATOR + strAppId;
    String unResolvedStageDir = STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR
            + strAppId;//from  w ww . j  av  a2s. co  m

    Path stagePath = new Path(unResolvedStageDir);
    FileSystem fs = stagePath.getFileSystem(conf);
    if (fs.exists(stagePath)) {
        fs.delete(stagePath, true);
    }
    Assert.assertFalse(fs.exists(stagePath));
    Path stageDir = TezCommonUtils.createTezSystemStagingPath(conf, strAppId);
    Assert.assertEquals(stageDir.toString(), expectedStageDir);
    Assert.assertTrue(fs.exists(stagePath));
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testTezSysStagingPath() throws Exception {
    String strAppId = "testAppId";
    Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    String expectedStageDir = RESOLVED_STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR
            + Path.SEPARATOR + strAppId;
    Assert.assertEquals(stageDir.toString(), expectedStageDir);
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testTezConfStagingPath() throws Exception {
    String strAppId = "testAppId";
    Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    Path confStageDir = TezCommonUtils.getTezConfStagingPath(stageDir);
    String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR
            + Path.SEPARATOR + strAppId + Path.SEPARATOR + TezConstants.TEZ_PB_BINARY_CONF_NAME;
    Assert.assertEquals(confStageDir.toString(), expectedDir);
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testTezSessionJarStagingPath() throws Exception {
    String strAppId = "testAppId";
    Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    Path confStageDir = TezCommonUtils.getTezAMJarStagingPath(stageDir);
    String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR
            + Path.SEPARATOR + strAppId + Path.SEPARATOR + TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME;
    Assert.assertEquals(confStageDir.toString(), expectedDir);
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testTezBinPlanStagingPath() throws Exception {
    String strAppId = "testAppId";
    Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    Path confStageDir = TezCommonUtils.getTezBinPlanStagingPath(stageDir);
    String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR
            + Path.SEPARATOR + strAppId + Path.SEPARATOR + TezConstants.TEZ_PB_PLAN_BINARY_NAME;
    Assert.assertEquals(confStageDir.toString(), expectedDir);
}

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testTezTextPlanStagingPath() throws Exception {
    String strAppId = "testAppId";
    String dagPBName = "testDagPBName";
    Path tezSysStagingPath = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
    Path confStageDir = TezCommonUtils.getTezTextPlanStagingPath(tezSysStagingPath, strAppId, dagPBName);
    String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR + TezCommonUtils.TEZ_SYSTEM_SUB_DIR
            + Path.SEPARATOR + strAppId + Path.SEPARATOR + strAppId + "-" + dagPBName + "-"
            + TezConstants.TEZ_PB_PLAN_TEXT_NAME;
    Assert.assertEquals(confStageDir.toString(), expectedDir);
}