Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testDuplicateJarsInDistributedCache() throws Exception {
    services = new Services();
    setSystemProps();/*from w  ww .  j a  v  a  2 s  .co m*/
    FileSystem fs = getFileSystem();
    Path basePath = new Path(
            services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
    Configuration conf = services.get(ConfigurationService.class).getConf();
    conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");

    // Use timedstamped directory if available
    Date time = new Date(System.currentTimeMillis());
    Path libpath = new Path(basePath,
            ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));

    Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
    createDirs(fs, pigPath, new Path(pigPath, "temp"));
    createFile(new Path(pigPath, "pig.jar"));
    createFile(new Path(pigPath, "hive.jar"));
    createFile(new Path(new Path(pigPath, "temp"), "pig.jar#pig.jar"));

    try {

        // DistributedCache should have only one pig jar
        verifyFilesInDistributedCache(setUpPigJob(true), "pig.jar", "hive.jar", "MyOozie.jar", "MyPig.jar");
        ShareLibService shareLibService = services.get(ShareLibService.class);
        // sharelib service should have two jars
        List<Path> shareLib = shareLibService.getShareLibJars("pig");
        assertEquals(shareLib.size(), 3);
        assertTrue(shareLib.toString().contains("pig.jar#pig.jar"));
        assertTrue(shareLib.toString().contains("hive.jar"));
    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

private void createTestShareLibMetaFile(FileSystem fs) {
    Properties prop = new Properties();

    try {//from   ww w. jav  a  2  s.  c om

        String testPath = "shareLibPath/";

        Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");

        Path somethingNew = new Path(testPath + Path.SEPARATOR + "something_new");
        fs.mkdirs(basePath);
        fs.mkdirs(somethingNew);

        createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
        createFile(somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR
                + "somethingNew.jar");

        prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString());
        prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new",
                "/user/test/" + somethingNew.toString());
        createTestShareLibMetaFile(fs, prop);

    } catch (IOException ex) {
        ex.printStackTrace();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

public void createTestShareLibMetaFile_multipleFile(FileSystem fs) {

    Properties prop = new Properties();
    try {/*from  w w  w  .  j av  a  2s . co  m*/

        String testPath = "shareLibPath/";

        Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
        Path somethingNew = new Path(testPath + Path.SEPARATOR + "something_new");
        fs.mkdirs(basePath);
        fs.mkdirs(somethingNew);

        createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
        createFile(somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR
                + "somethingNew.jar");

        prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig",
                "/user/test/" + basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar,"
                        + "/user/test/" + somethingNew.toString() + Path.SEPARATOR + "somethingNew"
                        + Path.SEPARATOR + "somethingNew.jar");

        FSDataOutputStream out = fs.create(new Path(metaFile));

        prop.store(out, null);
        out.close();

    } catch (IOException ex) {
        ex.printStackTrace();
    }
}

---

From source file:org.apache.oozie.tools.diag.TestAppInfoCollector.java

License:Apache License

@Test
public void testStoreLastWorkflows() throws Exception {
    final List<WorkflowJob> wfJobs = Arrays.asList(wfJob);
    doReturn(wfJobs).when(mockOozieClient).getJobsInfo(null, 0, 1);
    doReturn(wfJob).when(mockOozieClient).getJobInfo(anyString());

    final String wfName = "0000000-170926142250283-oozie-test-W";
    doReturn(wfName).when(wfJob).getId();
    doReturn("map-reduce-wf").when(wfJob).getAppName();
    doReturn("hdfs://localhost:9000/user/test/examples/apps/map-reduce/workflow.xml").when(wfJob).getAppPath();
    doReturn("test").when(wfJob).getUser();
    doReturn(null).when(wfJob).getAcl();
    doReturn(WorkflowJob.Status.SUCCEEDED).when(wfJob).getStatus();
    doReturn("").when(wfJob).getConsoleUrl();
    doReturn("http://0.0.0.0:11000/oozie?job=0000000-170926142250283-oozie-asas-W").when(wfJob).getExternalId();

    doReturn(null).when(wfJob).getParentId();

    final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss zzz");
    final Date startDate = formatter.parse("2017-09-26 12:22:57 GMT");
    doReturn(startDate).when(wfJob).getCreatedTime();

    final Date endDate = formatter.parse("2017-09-26 12:32:57 GMT");
    doReturn(endDate).when(wfJob).getEndTime();
    doReturn(endDate).when(wfJob).getLastModifiedTime();
    doReturn(startDate).when(wfJob).getStartTime();
    doReturn(0).when(wfJob).getRun();//from w  w  w. j  a v  a 2 s. com

    doReturn("0000000-170926142250283-oozie-asas-W@:start:").when(wfAction).getId();
    doReturn(":start:").when(wfAction).getName();
    doReturn(":START:").when(wfAction).getType();
    doReturn(WorkflowAction.Status.OK).when(wfAction).getStatus();
    doReturn("mr-node").when(wfAction).getTransition();
    doReturn(startDate).when(wfAction).getStartTime();
    doReturn(endDate).when(wfAction).getEndTime();
    doReturn(null).when(wfAction).getErrorCode();
    doReturn(null).when(wfAction).getErrorMessage();
    doReturn("").when(wfAction).getConsoleUrl();
    doReturn("").when(wfAction).getTrackerUri();
    doReturn(null).when(wfAction).getExternalChildIDs();
    doReturn("").when(wfAction).getExternalId();
    doReturn("OK").when(wfAction).getExternalStatus();
    doReturn(null).when(wfAction).getData();
    doReturn(null).when(wfAction).getStats();
    doReturn(null).when(wfAction).getCred();
    doReturn(0).when(wfAction).getRetries();
    doReturn(10).when(wfAction).getUserRetryInterval();
    doReturn(0).when(wfAction).getUserRetryCount();
    doReturn(0).when(wfAction).getUserRetryMax();

    final List<WorkflowAction> wfActions = Arrays.asList(wfAction);
    doReturn(wfActions).when(wfJob).getActions();

    appInfoCollector.storeLastWorkflows(testFolder, 1, 1);

    final File infoOut = new File(testFolder, wfName + Path.SEPARATOR + "info.txt");
    assertTrue(infoOut.exists());
    assertFileContains(infoOut, wfName);
}

---

From source file:org.apache.oozie.tools.diag.TestAppInfoCollector.java

License:Apache License

@Test
public void testStoreCoordinators() throws Exception {
    final List<CoordinatorJob> coordJobs = Arrays.asList(coordJob);
    doReturn(coordJobs).when(mockOozieClient).getCoordJobsInfo(null, 0, 1);

    final String coordId = "0000000-170926142250283-oozie-test-C";
    doReturn(coordId).when(coordJob).getId();
    doReturn(Job.Status.RUNNING).when(coordJob).getStatus();
    doReturn(CoordinatorJob.Execution.FIFO).when(coordJob).getExecutionOrder();

    final List<CoordinatorAction> coordinatorActions = Arrays.asList(coordinatorAction);
    doReturn(CoordinatorAction.Status.KILLED).when(coordinatorAction).getStatus();
    doReturn(coordinatorActions).when(coordJob).getActions();
    doReturn(coordJob).when(mockOozieClient).getCoordJobInfo(anyString());
    doReturn(CoordinatorJob.Timeunit.MINUTE).when(coordJob).getTimeUnit();

    appInfoCollector.storeLastCoordinators(testFolder, 1, 1);

    final File coordInfoOut = new File(testFolder, coordId + Path.SEPARATOR + "info.txt");
    assertTrue(coordInfoOut.exists());//from  w w w . ja va2  s.  c om
    assertFileContains(coordInfoOut, coordId);
}

---

From source file:org.apache.oozie.tools.diag.TestAppInfoCollector.java

License:Apache License

@Test
public void testStoreLastBundles() throws Exception {
    final List<BundleJob> bundleJobs = Arrays.asList(bundleJob);
    doReturn(bundleJobs).when(mockOozieClient).getBundleJobsInfo(null, 0, 1);

    final String bundleId = "0000027-110322105610515-oozie-chao-B";
    doReturn(bundleId).when(bundleJob).getId();
    doReturn(Job.Status.RUNNING).when(bundleJob).getStatus();
    doReturn(bundleJob).when(mockOozieClient).getBundleJobInfo(anyString());

    appInfoCollector.storeLastBundles(testFolder, 1, 1);

    final File bundleInfoOut = new File(testFolder, bundleId + Path.SEPARATOR + "info.txt");
    assertTrue(bundleInfoOut.exists());/*from  ww w  .ja va2 s .co  m*/
    assertFileContains(bundleInfoOut, bundleId);
}

---

From source file:org.apache.oozie.tools.IntegrationTestOozieSharelibCLI.java

License:Apache License

public void testOozieSharelibCreateExtraLibs() throws Exception {
    File extraLibBirectory1 = tmpFolder.newFolder("extralib1");
    File extraLibBirectory2 = tmpFolder.newFolder("extralib2");
    final int sharelibFileNr = 3;
    final int extraSharelib1FileNr = 4;
    final int extraSharelib2FileNr = 4;
    List<File> shareLibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, sharelibFileNr);
    List<File> extraShareLibFiles1 = OozieSharelibFileOperations.generateAndWriteFiles(extraLibBirectory1,
            extraSharelib1FileNr);// www. j ava  2s.c om
    List<File> extraShareLibFiles2 = OozieSharelibFileOperations.generateAndWriteFiles(extraLibBirectory2,
            extraSharelib2FileNr);

    String extraLib1 = extraLibBirectory1.getName() + EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR
            + extraLibBirectory1.getAbsolutePath();
    String extraLib2 = extraLibBirectory2.getName() + EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR
            + extraLibBirectory2.getAbsolutePath();
    String[] argsCreate = { "create", "-fs", outPath, "-locallib",
            libDirectory.getParentFile().getAbsolutePath(), "-" + OozieSharelibCLI.EXTRALIBS, extraLib1,
            "-" + OozieSharelibCLI.EXTRALIBS, extraLib2 };

    assertEquals("Exit code mismatch", 0, execOozieSharelibCLICommands(argsCreate));

    ShareLibService sharelibService = getServices().get(ShareLibService.class);
    Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), ShareLibService.SHARE_LIB_PREFIX);
    Path extraSharelibPath1 = new Path(latestLibPath + Path.SEPARATOR + extraLibBirectory1.getName());
    Path extraSharelibPath2 = new Path(latestLibPath + Path.SEPARATOR + extraLibBirectory2.getName());

    checkCopiedSharelibFiles(shareLibFiles, latestLibPath);
    checkCopiedSharelibFiles(extraShareLibFiles1, extraSharelibPath1);
    checkCopiedSharelibFiles(extraShareLibFiles2, extraSharelibPath2);
}

---

From source file:org.apache.oozie.tools.OozieSharelibCLI.java

License:Apache License

public synchronized int run(String[] args) throws Exception {
    if (used) {//w w  w  .  j av a  2s  .  c o m
        throw new IllegalStateException("CLI instance already used");
    }

    used = true;

    CLIParser parser = new CLIParser("oozie-setup.sh", HELP_INFO);
    String oozieHome = System.getProperty(OOZIE_HOME);
    parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(),
            false);
    parser.addCommand(CREATE_CMD, "", "create a new timestamped version of oozie sharelib",
            createUpgradeOptions(CREATE_CMD), false);
    parser.addCommand(UPGRADE_CMD, "",
            "[deprecated][use command \"create\" to create new version]   upgrade oozie sharelib \n",
            createUpgradeOptions(UPGRADE_CMD), false);

    try {
        final CLIParser.Command command = parser.parse(args);
        String sharelibAction = command.getName();

        if (sharelibAction.equals(HELP_CMD)) {
            parser.showHelp(command.getCommandLine());
            return 0;
        }

        if (!command.getCommandLine().hasOption(FS_OPT)) {
            throw new Exception("-fs option must be specified");
        }

        int threadPoolSize = Integer.valueOf(command.getCommandLine().getOptionValue(CONCURRENCY_OPT, "1"));
        File srcFile = null;

        //Check whether user provided locallib
        if (command.getCommandLine().hasOption(LIB_OPT)) {
            srcFile = new File(command.getCommandLine().getOptionValue(LIB_OPT));
        } else {
            //Since user did not provide locallib, find the default one under oozie home dir
            Collection<File> files = FileUtils.listFiles(new File(oozieHome),
                    new WildcardFileFilter("oozie-sharelib*.tar.gz"), null);

            if (files.size() > 1) {
                throw new IOException("more than one sharelib tar found at " + oozieHome);
            }

            if (files.isEmpty()) {
                throw new IOException("default sharelib tar not found in oozie home dir: " + oozieHome);
            }

            srcFile = files.iterator().next();
        }

        File temp = File.createTempFile("oozie", ".dir");
        temp.delete();
        temp.mkdir();
        temp.deleteOnExit();

        //Check whether the lib is a tar file or folder
        if (!srcFile.isDirectory()) {
            FileUtil.unTar(srcFile, temp);
            srcFile = new File(temp.toString() + "/share/lib");
        } else {
            //Get the lib directory since it's a folder
            srcFile = new File(srcFile, "lib");
        }

        String hdfsUri = command.getCommandLine().getOptionValue(FS_OPT);
        Path srcPath = new Path(srcFile.toString());

        Services services = new Services();
        services.getConf().set(Services.CONF_SERVICE_CLASSES,
                "org.apache.oozie.service.LiteWorkflowAppService, org.apache.oozie.service.HadoopAccessorService");
        services.getConf().set(Services.CONF_SERVICE_EXT_CLASSES, "");
        services.init();
        WorkflowAppService lwas = services.get(WorkflowAppService.class);
        HadoopAccessorService has = services.get(HadoopAccessorService.class);
        Path dstPath = lwas.getSystemLibPath();

        if (sharelibAction.equals(CREATE_CMD) || sharelibAction.equals(UPGRADE_CMD)) {
            dstPath = new Path(
                    dstPath.toString() + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory());
        }

        System.out.println("the destination path for sharelib is: " + dstPath);

        URI uri = new Path(hdfsUri).toUri();
        Configuration fsConf = has.createJobConf(uri.getAuthority());
        FileSystem fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);

        if (!srcFile.exists()) {
            throw new IOException(srcPath + " cannot be found");
        }

        if (threadPoolSize > 1) {
            concurrentCopyFromLocal(fs, threadPoolSize, srcFile, dstPath);
        } else {
            fs.copyFromLocalFile(false, srcPath, dstPath);
        }

        services.destroy();
        FileUtils.deleteDirectory(temp);

        return 0;
    } catch (ParseException ex) {
        System.err.println("Invalid sub-command: " + ex.getMessage());
        System.err.println();
        System.err.println(parser.shortHelp());
        return 1;
    } catch (Exception ex) {
        logError(ex.getMessage(), ex);
        return 1;
    }
}

---

From source file:org.apache.oozie.util.ClasspathUtils.java

License:Apache License

private static void addToClasspathIfNotJar(Path[] paths, URI[] withLinks, Configuration conf,
        Map<String, String> environment, String classpathEnvVar) throws IOException {
    if (paths != null) {
        HashMap<Path, String> linkLookup = new HashMap<Path, String>();
        if (withLinks != null) {
            for (URI u : withLinks) {
                Path p = new Path(u);
                FileSystem remoteFS = p.getFileSystem(conf);
                p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
                String name = (null == u.getFragment()) ? p.getName() : u.getFragment();
                if (!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) {
                    linkLookup.put(p, name);
                }//from ww  w. j ava2s  .c o  m
            }
        }

        for (Path p : paths) {
            FileSystem remoteFS = p.getFileSystem(conf);
            p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()));
            String name = linkLookup.get(p);
            if (name == null) {
                name = p.getName();
            }
            if (!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) {
                MRApps.addToEnvironment(environment, classpathEnvVar,
                        ApplicationConstants.Environment.PWD.$() + Path.SEPARATOR + name, conf);
            }
        }
    }
}

---

From source file:org.apache.phoenix.end2end.index.LocalIndexIT.java

License:Apache License

private void copyLocalIndexHFiles(Configuration conf, HRegionInfo fromRegion, HRegionInfo toRegion,
        boolean move) throws IOException {
    Path root = FSUtils.getRootDir(conf);

    Path seondRegion = new Path(HTableDescriptor.getTableDir(root, fromRegion.getTableName()) + Path.SEPARATOR
            + fromRegion.getEncodedName() + Path.SEPARATOR + "L#0/");
    Path hfilePath = FSUtils.getCurrentFileSystem(conf).listFiles(seondRegion, true).next().getPath();
    Path firstRegionPath = new Path(HTableDescriptor.getTableDir(root, toRegion.getTableName()) + Path.SEPARATOR
            + toRegion.getEncodedName() + Path.SEPARATOR + "L#0/");
    FileSystem currentFileSystem = FSUtils.getCurrentFileSystem(conf);
    assertTrue(FileUtil.copy(currentFileSystem, hfilePath, currentFileSystem, firstRegionPath, move, conf));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »