Example usage for org.apache.hadoop.fs Path SEPARATOR

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path SEPARATOR.

Prototype

String SEPARATOR

To view the source code for org.apache.hadoop.fs Path SEPARATOR.

Click Source Link

Document

The directory separator, a slash.

Usage

From source file:org.apache.oozie.service.TestShareLibMappingFileInput.java

License:Apache License

public TestShareLibMappingFileInput(final FileSystem fs, final String sharelibMappingActionName,
        final String sharelibPath) {
    this.fs = fs;
    this.sharelibName = sharelibMappingActionName;
    this.sharelibNameWithMappingFilePrefix = ShareLibService.SHARE_LIB_CONF_PREFIX + "."
            + sharelibMappingActionName;
    this.sharelibPath = sharelibPath;
    this.baseName = sharelibPath.substring(sharelibPath.lastIndexOf(Path.SEPARATOR) + 1);
}

---

From source file:org.apache.oozie.service.TestShareLibMappingFileInput.java

License:Apache License

String getFullShareLibPathDir() {
    String fullShareLibPathDir = getLocalizedShareLibPath(fs, sharelibPath);
    return fullShareLibPathDir.substring(0, fullShareLibPathDir.lastIndexOf(Path.SEPARATOR));
}

---

From source file:org.apache.oozie.service.TestShareLibMappingFileInput.java

License:Apache License

@Override
String getFullShareLibPathDir() {
    int lastIndexOfPathSeparator = sharelibPath.lastIndexOf(Path.SEPARATOR);
    String symlinkTargetFileName = (lastIndexOfPathSeparator != -1)
            ? sharelibPath.substring(lastIndexOfPathSeparator + 1)
            : sharelibPath;/*from  w ww  . ja va2s .c  om*/
    String fullShareLibPathDir = getLocalizedShareLibPath(fs, symlinkPath);
    return fullShareLibPathDir + "#" + symlinkTargetFileName;
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testAddShareLib_pig_withVersion() throws Exception {
    services = new Services();
    setSystemProps();//from w w  w.  j av  a 2 s  .co  m
    Configuration conf = services.get(ConfigurationService.class).getConf();
    conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");

    FileSystem fs = getFileSystem();
    Date time = new Date(System.currentTimeMillis());

    Path basePath = new Path(
            services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
    Path libpath = new Path(basePath,
            ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));
    fs.mkdirs(libpath);

    Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
    Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
    Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
    fs.mkdirs(pigPath);
    fs.mkdirs(pigPath1);
    fs.mkdirs(pigPath2);

    createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");

    try {
        services.init();
        String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
                + getNameNodeUri() + "</name-node>"
                + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>"
                + "</pig>";
        Element eActionXml = XmlUtils.parseXml(actionXml);
        XConfiguration protoConf = new XConfiguration();
        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
        WorkflowJobBean wfj = new WorkflowJobBean();
        protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
        wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());

        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
        PigActionExecutor ae = new PigActionExecutor();
        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
        jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);

        verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "MyPig.jar", "MyOozie.jar",
                "pig-10.jar");

    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testGetShareLibCompatible() throws Exception {
    services = new Services();
    setSystemProps();/* w ww.  j a v a 2s. co m*/
    FileSystem fs = getFileSystem();
    Path basePath = new Path(
            services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));

    // Use basepath if there is no timestamped directory
    fs.mkdirs(basePath);
    Path pigPath = new Path(basePath.toString() + Path.SEPARATOR + "pig");
    fs.mkdirs(pigPath);
    try {
        services.init();
        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
        assertNotNull(shareLibService.getShareLibJars("pig"));
    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testGetShareLibPath() throws Exception {
    services = new Services();
    setSystemProps();/* w w w  .  j  av a 2  s  . c  om*/
    FileSystem fs = getFileSystem();
    Path basePath = new Path(
            services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));

    // Use timedstamped directory if available
    Date time = new Date(System.currentTimeMillis());
    Path libpath = new Path(basePath,
            ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));
    fs.mkdirs(libpath);

    Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
    Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
    Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
    fs.mkdirs(pigPath);
    fs.mkdirs(pigPath1);
    fs.mkdirs(pigPath2);
    try {
        services.init();
        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
        assertNotNull(shareLibService.getShareLibJars("pig"));
        assertNotNull(shareLibService.getShareLibJars("pig_9"));
        assertNotNull(shareLibService.getShareLibJars("pig_10"));
        assertNull(shareLibService.getShareLibJars("pig_11"));
    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testShareLib() throws Exception {
    services = new Services();
    setSystemProps();/*from  w w  w. j  av a 2  s. c o  m*/
    Configuration conf = services.get(ConfigurationService.class).getConf();
    conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
    FileSystem fs = getFileSystem();
    String dir1 = dt
            .format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
    String dir2 = dt
            .format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
    String dir3 = dt
            .format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(3, TimeUnit.DAYS)));
    Path basePath = new Path(
            services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
    Path path1 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + dir1);
    Path path2 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + dir2);
    Path path3 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + dir3);
    createDirs(fs, path1, path2, path3);
    createFile(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
    try {
        services.init();
        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
        assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testLoadfromDFS() throws Exception {
    services = new Services();
    setSystemProps();/*from  w ww.  j ava2s  .c  o m*/
    try {
        services.init();
        FileSystem fs = getFileSystem();
        Date time = new Date(System.currentTimeMillis());

        Path basePath = new Path(
                services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
        Path libpath = new Path(basePath,
                ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));
        fs.mkdirs(libpath);

        Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
        Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
        Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
        Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
        fs.mkdirs(pigPath);
        fs.mkdirs(ooziePath);
        fs.mkdirs(pigPath1);
        fs.mkdirs(pigPath2);

        createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
        createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");

        String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
                + getNameNodeUri() + "</name-node>"
                + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>"
                + "</pig>";
        Element eActionXml = XmlUtils.parseXml(actionXml);
        XConfiguration protoConf = new XConfiguration();
        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
        WorkflowJobBean wfj = new WorkflowJobBean();
        protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
        wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());

        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
        PigActionExecutor ae = new PigActionExecutor();
        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
        jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);

        verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "pig-10.jar",
                "oozie_luncher.jar");

    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testMultipleLauncherCall() throws Exception {
    services = new Services();
    setSystemProps();//ww  w.  j a v  a2 s  .  c  om
    Configuration conf = services.get(ConfigurationService.class).getConf();
    conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
    try {
        FileSystem fs = getFileSystem();
        Date time = new Date(System.currentTimeMillis());
        Path basePath = new Path(
                services.get(ConfigurationService.class).getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
        Path libpath = new Path(basePath,
                ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));
        fs.mkdirs(libpath);
        Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
        fs.mkdirs(ooziePath);
        createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
        services.init();
        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
        List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
        assertEquals(launcherPath.size(), 2);
        launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
        assertEquals(launcherPath.size(), 2);
    } finally {
        services.destroy();
    }
}

---

From source file:org.apache.oozie.service.TestShareLibService.java

License:Apache License

@Test
public void testMetafileSymlink() throws ServiceException, IOException {
    // Assume.assumeTrue("Skipping for hadoop - 1.x",HadoopFileSystem.isSymlinkSupported());
    if (!HadoopShims.isSymlinkSupported()) {
        return;/* w  w  w.  j  a  va2s  . c  om*/
    }

    services = new Services();
    setSystemProps();
    Configuration conf = services.get(ConfigurationService.class).getConf();
    conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
    services.init();
    FileSystem fs = getFileSystem();
    Properties prop = new Properties();
    try {

        String testPath = "shareLibPath/";

        Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
        Path basePath1 = new Path(testPath + Path.SEPARATOR + "testPath1");
        Path hive_site = new Path(
                basePath.toString() + Path.SEPARATOR + "hive_conf" + Path.SEPARATOR + "hive-site.xml");
        Path hive_site1 = new Path(
                basePath.toString() + Path.SEPARATOR + "hive_conf" + Path.SEPARATOR + "hive-site1.xml");
        Path symlink = new Path("symlink/");
        Path symlink_hive_site = new Path("symlink/hive_conf" + Path.SEPARATOR + "hive-site.xml");

        fs.mkdirs(basePath);

        createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
        createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_1.jar");

        createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_2.jar");
        createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_3.jar");
        createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_4.jar");

        createFile(hive_site.toString());

        HadoopShims fileSystem = new HadoopShims(fs);
        fileSystem.createSymlink(basePath, symlink, true);
        fileSystem.createSymlink(hive_site, symlink_hive_site, true);

        prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + symlink.toString());
        prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".hive_conf",
                "/user/test/" + symlink_hive_site.toString() + "#hive-site.xml");
        createTestShareLibMetaFile(fs, prop);
        assertEquals(fileSystem.isSymlink(symlink), true);

        conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
        try {
            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
            assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
            assertEquals(shareLibService.getShareLibJars("hive_conf").size(), 1);
            new HadoopShims(fs).createSymlink(basePath1, symlink, true);
            new HadoopShims(fs).createSymlink(hive_site1, symlink_hive_site, true);
            assertEquals(
                    new HadoopShims(fs).getSymLinkTarget(shareLibService.getShareLibJars("hive_conf").get(0)),
                    hive_site1);
            assertEquals(shareLibService.getShareLibJars("pig").size(), 3);
        } finally {
            fs.delete(new Path("shareLibPath/"), true);
            fs.delete(new Path(metaFile), true);
            fs.delete(new Path("/user/test/config.properties"), true);

            fs.delete(symlink, true);
            services.destroy();
        }
    } catch (IOException ex) {
        ex.printStackTrace();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »