Example usage for org.apache.hadoop.fs FileContext mkdir

List of usage examples for org.apache.hadoop.fs FileContext mkdir

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileContext
  6. mkdir

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileContext mkdir.

Prototype

public void mkdir(final Path dir, final FsPermission permission, final boolean createParent)
        throws AccessControlException, FileAlreadyExistsException, FileNotFoundException,
        ParentNotDirectoryException, UnsupportedFileSystemException, IOException

Source Link

Document

Make(create) a directory and all the non-existent parents.

Usage

From source file:com.cloudera.RenameTest.java

License:Apache License

public static void testFileContextRename(URI uri) throws Exception {
    FileContext fc = FileContext.getFileContext(uri);
    Path testDir = new Path(new Path(uri), "testdir");
    System.out.println("mkdir " + testDir);
    fc.mkdir(testDir, new FsPermission((short) 0755), true);
    Path testFile = new Path(new Path(uri), "testfile");
    System.out.println("create " + testFile);
    FSDataOutputStream fos = fc.create(testFile, EnumSet.of(CREATE));
    fos.close();//from ww  w.j ava  2  s.  co m
    System.out.println("rename " + testFile + " -> " + testDir);
    fc.rename(testFile, testDir);
}

From source file:com.ikanow.aleph2.core.shared.utils.DirUtils.java

License:Apache License

/** Creates a directory in the storage servce
 * @param fileContext//from   w w  w.j a  va 2 s . c  o  m
 * @param pathString
 */
public static void createDirectory(FileContext fileContext, String pathString) {
    if (fileContext != null && pathString != null) {
        try {
            Path dir = new Path(pathString);
            if (!fileContext.util().exists(dir)) {
                fileContext.mkdir(dir, DEFAULT_DIR_PERMS, true); //(note perm is & with umask)
                try {
                    fileContext.setPermission(dir, DEFAULT_DIR_PERMS);
                } catch (Exception e) {
                } // (not supported in all FS)
            }
        } catch (Exception e) {
            logger.error("createFolderStructure Caught Exception", e);
        }
    }

}

From source file:com.ikanow.aleph2.management_db.services.DataBucketCrudService.java

License:Apache License

/** Create the different default files needed by the bucket in the distributed file system
 * @param bucket/*from   w  ww.j  a va2s.  co m*/
 * @param storage_service
 * @throws Exception
 */
public static void createFilePaths(final DataBucketBean bucket, final IStorageService storage_service)
        throws Exception {
    final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty())
            .get();

    final String bucket_root = storage_service.getBucketRootPath() + "/" + bucket.full_name();

    // Check if a "delete touch file is present, bail if so"
    if (doesBucketPathExist(bucket, storage_service, Optional.of(DELETE_TOUCH_FILE))) {
        throw new RuntimeException(
                ErrorUtils.get(ManagementDbErrorUtils.DELETE_TOUCH_FILE_PRESENT, bucket.full_name()));
    }

    Arrays.asList("", "logs", "logs/harvest", "logs/enrichment", "logs/storage", "assets", "import",
            "import/temp", "import/temp/upload", "import/transient", "import/stored", "import/stored/raw",
            "import/stored/json", "import/stored/processed", "import/ready").stream()
            .map(s -> new Path(bucket_root + IStorageService.BUCKET_SUFFIX + s))
            .forEach(Lambdas.wrap_consumer_u(p -> {
                dfs.mkdir(p, DEFAULT_DIR_PERMS, true); //(note perm is & with umask)
                try {
                    dfs.setPermission(p, DEFAULT_DIR_PERMS);
                } catch (Exception e) {
                } // (not supported in all FS)
            }));
}

From source file:com.ikanow.aleph2.storage_service_hdfs.services.TestMockHdfsStorageSystem.java

License:Apache License

@Test
public void test_basic_secondaryBuffers()
        throws AccessControlException, FileAlreadyExistsException, FileNotFoundException,
        ParentNotDirectoryException, UnsupportedFileSystemException, IllegalArgumentException, IOException {
    // 0) Setup/*from  w w w  .  j  a v  a  2 s  . c  o m*/
    final String temp_dir = System.getProperty("java.io.tmpdir") + File.separator;

    final GlobalPropertiesBean globals = BeanTemplateUtils.build(GlobalPropertiesBean.class)
            .with(GlobalPropertiesBean::local_yarn_config_dir, temp_dir)
            .with(GlobalPropertiesBean::distributed_root_dir, temp_dir)
            .with(GlobalPropertiesBean::local_root_dir, temp_dir)
            .with(GlobalPropertiesBean::distributed_root_dir, temp_dir).done().get();

    final MockHdfsStorageService storage_service = new MockHdfsStorageService(globals);

    // Some buckets

    final DataBucketBean bucket = BeanTemplateUtils.build(DataBucketBean.class)
            .with(DataBucketBean::full_name, "/test/storage/bucket")
            .with(DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done().get();

    setup_bucket(storage_service, bucket, Collections.emptyList());

    // Get primary buffer doesn't work:

    assertFalse(storage_service.getDataService().get().getPrimaryBufferName(bucket).isPresent());

    // Add some secondary buffers and check they get picked up

    final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty())
            .get();
    final String bucket_root = storage_service.getBucketRootPath() + "/" + bucket.full_name();
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "test1"),
            FsPermission.getDirDefault(), true);
    //(skip the current dir once just to check it doesn't cause problems)
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test2"),
            FsPermission.getDirDefault(), true);
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON), FsPermission.getDirDefault(),
            true);
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "test3"),
            FsPermission.getDirDefault(), true);
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED),
            FsPermission.getDirDefault(), true);

    assertEquals(Arrays.asList("test1", "test2", "test3"), storage_service.getDataService().get()
            .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList()));

    //(check dedups)
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test1"),
            FsPermission.getDirDefault(), true);

    assertEquals(Arrays.asList("test1", "test2", "test3"), storage_service.getDataService().get()
            .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList()));

    try {
        dfs.delete(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "test3"),
                true);
    } catch (Exception e) {
    }

    assertEquals(Arrays.asList("test1", "test2"), storage_service.getDataService().get()
            .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList()));

    try {
        dfs.delete(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "test1"), true);
    } catch (Exception e) {
    }

    assertEquals(Arrays.asList("test1", "test2"), storage_service.getDataService().get()
            .getSecondaryBuffers(bucket).stream().sorted().collect(Collectors.toList()));
}

From source file:com.ikanow.aleph2.storage_service_hdfs.services.TestMockHdfsStorageSystem.java

License:Apache License

@Test
public void test_switching_secondaryBuffers() throws AccessControlException, FileAlreadyExistsException,
        FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException,
        IllegalArgumentException, IOException, InterruptedException, ExecutionException {
    // 0) Setup/*from ww w .  j av  a2 s  .  c o m*/
    final String temp_dir = System.getProperty("java.io.tmpdir") + File.separator;

    final GlobalPropertiesBean globals = BeanTemplateUtils.build(GlobalPropertiesBean.class)
            .with(GlobalPropertiesBean::local_yarn_config_dir, temp_dir)
            .with(GlobalPropertiesBean::distributed_root_dir, temp_dir)
            .with(GlobalPropertiesBean::local_root_dir, temp_dir)
            .with(GlobalPropertiesBean::distributed_root_dir, temp_dir).done().get();

    final MockHdfsStorageService storage_service = new MockHdfsStorageService(globals);

    // Some buckets

    final DataBucketBean bucket = BeanTemplateUtils.build(DataBucketBean.class)
            .with(DataBucketBean::full_name, "/test/storage/bucket")
            .with(DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done().get();

    setup_bucket(storage_service, bucket, Collections.emptyList());

    final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty())
            .get();
    final String bucket_root = storage_service.getBucketRootPath() + "/" + bucket.full_name();
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW + "test_exdir"),
            FsPermission.getDirDefault(), true);
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "test1"),
            FsPermission.getDirDefault(), true);
    //(skip the current dir once just to check it doesn't cause problems)
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test2"),
            FsPermission.getDirDefault(), true);
    dfs.create(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test2/test2.json"),
            EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)).close();
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON + "test_exdir"),
            FsPermission.getDirDefault(), true);
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "test3"),
            FsPermission.getDirDefault(), true);
    dfs.mkdir(new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED + "test_exdir"),
            FsPermission.getDirDefault(), true);

    // (retire the primary, copy test2 across)
    {
        BasicMessageBean res1 = storage_service.getDataService().get()
                .switchCrudServiceToPrimaryBuffer(bucket, Optional.of("test2"), Optional.empty()).get();
        System.out.println("(res1 = " + res1.message() + ")");
        assertTrue("Request returns: " + res1.message(), res1.success());
    }
    assertTrue(doesDirExist(dfs, new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON)));
    assertTrue(
            doesFileExist(dfs, new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON + "test2.json")));
    assertTrue(doesDirExist(dfs, new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW)));
    assertTrue(doesDirExist(dfs, new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED)));
    assertFalse(doesDirExist(dfs,
            new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "test2")));
    assertFalse(doesDirExist(dfs,
            new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "test2")));
    assertFalse(doesDirExist(dfs,
            new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "test2")));
    assertTrue(doesDirExist(dfs, new Path(
            bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "former_current/test_exdir")));
    assertTrue(doesDirExist(dfs, new Path(
            bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "former_current/test_exdir")));
    assertTrue(doesDirExist(dfs, new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY
            + "former_current/test_exdir")));
    {
        BasicMessageBean res2 = storage_service.getDataService().get()
                .switchCrudServiceToPrimaryBuffer(bucket, Optional.of("test3"), Optional.of("ex_primary"))
                .get();
        System.out.println("(res2 = " + res2.message() + ")");
        assertTrue("Request returns: " + res2.message(), res2.success());
    }
    assertTrue(doesDirExist(dfs,
            new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "ex_primary")));
    assertTrue(doesDirExist(dfs,
            new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW_SECONDARY + "ex_primary")));
    assertTrue(doesDirExist(dfs,
            new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED_SECONDARY + "ex_primary")));
    assertTrue(doesFileExist(dfs, new Path(
            bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON_SECONDARY + "ex_primary/test2.json")));

    // return to the primary, delete the current
    {
        BasicMessageBean res3 = storage_service.getDataService().get()
                .switchCrudServiceToPrimaryBuffer(bucket, Optional.of("former_current"), Optional.of("")).get();
        System.out.println("(res3 = " + res3.message() + ")");
        assertTrue("Request returns: " + res3.message(), res3.success());
        assertTrue(doesDirExist(dfs,
                new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_JSON + "/test_exdir")));
        assertTrue(doesDirExist(dfs,
                new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_RAW + "/test_exdir")));
        assertTrue(doesDirExist(dfs,
                new Path(bucket_root + IStorageService.STORED_DATA_SUFFIX_PROCESSED + "/test_exdir")));
    }
}

From source file:com.ikanow.aleph2.storage_service_hdfs.services.TestMockHdfsStorageSystem.java

License:Apache License

/**
 * @param storage_service//ww  w  .  jav a2 s  . c om
 * @param bucket
 * @param extra_suffixes - start with $ to indicate a new secondary buffer, else is a normal suffix 
 */
protected void setup_bucket(MockHdfsStorageService storage_service, final DataBucketBean bucket,
        List<String> extra_suffixes) {
    final FileContext dfs = storage_service.getUnderlyingPlatformDriver(FileContext.class, Optional.empty())
            .get();

    final String bucket_root = storage_service.getBucketRootPath() + "/" + bucket.full_name();

    //(first delete root path)
    try {
        dfs.delete(new Path(bucket_root), true);
    } catch (Exception e) {
    }

    Stream.concat(
            Arrays.asList("/managed_bucket", "/managed_bucket/logs", "/managed_bucket/logs/harvest",
                    "/managed_bucket/logs/enrichment", "/managed_bucket/logs/storage", "/managed_bucket/assets",
                    "/managed_bucket/import", "/managed_bucket/import/stored",
                    "/managed_bucket/import/stored/raw/current", "/managed_bucket/import/stored/json/current",
                    "/managed_bucket/import/stored/processed/current",
                    "/managed_bucket/import/transient/current", "/managed_bucket/import/ready",
                    "/managed_bucket/import/temp").stream(),
            extra_suffixes.stream()
                    .flatMap(
                            s -> s.startsWith("$")
                                    ? Stream.of("/managed_bucket/import/stored/raw/" + s.substring(1),
                                            "/managed_bucket/import/stored/json/" + s.substring(1),
                                            "/managed_bucket/import/stored/processed/" + s.substring(1),
                                            "/managed_bucket/import/transient/" + s.substring(1))
                                    : Stream.of(s)))
            .map(s -> new Path(bucket_root + s))
            .forEach(Lambdas.wrap_consumer_u(p -> dfs.mkdir(p, FsPermission.getDefault(), true)));
}

From source file:org.apache.solr.hadoop.hack.MiniMRYarnCluster.java

License:Apache License

@Override
public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath());
    }//from  w  w  w .  j  av a  2  s .  com

    // By default, VMEM monitoring disabled, PMEM monitoring enabled.
    if (!conf.getBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
            MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
        conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
        conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
    }

    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
        Path stagingPath = FileContext.getFileContext(conf)
                .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        /*
         * Re-configure the staging path on Windows if the file system is localFs.
         * We need to use a absolute path that contains the drive letter. The unit
         * test could run on a different drive than the AM. We can run into the
         * issue that job files are localized to the drive where the test runs on,
         * while the AM starts on a different drive and fails to find the job
         * metafiles. Using absolute path can avoid this ambiguity.
         */
        if (Path.WINDOWS) {
            if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
                conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                        new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath());
            }
        }
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        //mkdir the staging directory so that right permissions are set while running as proxy user
        fc.mkdir(stagingPath, null, true);
        //mkdir done directory as well 
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new YarnRuntimeException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
                                               // which shuffle doesn't happen
                                               //configure the shuffle service in NM
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
            new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
    conf.setClass(String.format(Locale.ENGLISH, YarnConfiguration.NM_AUX_SERVICE_FMT,
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class);

    // Non-standard shuffle port
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class,
            ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

    super.serviceInit(conf);
}

From source file:org.apache.tajo.MiniTajoYarnCluster.java

License:Apache License

@Override
public void init(Configuration conf) {

    conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, new InetSocketAddress("127.0.0.1", 0));
    conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, new InetSocketAddress("127.0.0.1", 0));

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath());
    }/*from  w ww .j  a v a  2  s  .c  om*/
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
        Path stagingPath = FileContext.getFileContext(conf)
                .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        //mkdir the staging directory so that right permissions are set while running as proxy user
        fc.mkdir(stagingPath, null, true);
        //mkdir done directory as well
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new YarnRuntimeException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
    // which shuffle doesn't happen
    //configure the shuffle service in NM
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, PullServerAuxService.PULLSERVER_SERVICEID);
    conf.setClass(
            String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, PullServerAuxService.PULLSERVER_SERVICEID),
            PullServerAuxService.class, Service.class);

    // Non-standard shuffle port
    conf.setInt(TajoConf.ConfVars.PULLSERVER_PORT.name(), 0);

    // local directory
    conf.set(TajoConf.ConfVars.WORKER_TEMPORAL_DIR.name(), "/tmp/tajo-localdir");

    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class,
            ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

    conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 600);

    super.init(conf);
}

From source file:org.apache.tez.mapreduce.MiniMRRTezCluster.java

License:Apache License

@Override
public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                new File(getTestWorkDir(), "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
    }//from  ww  w .  j  a  v  a2  s.c o m

    // VMEM monitoring disabled, PMEM monitoring enabled.
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);

    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
        Path stagingPath = FileContext.getFileContext(conf)
                .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        fc.mkdir(stagingPath, null, true);

        //mkdir done directory as well
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new TezUncheckedException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test");

    //configure the shuffle service in NM
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
            new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
    conf.setClass(
            String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID),
            ShuffleHandler.class, Service.class);

    // Non-standard shuffle port
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class,
            ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    super.serviceInit(conf);
}

From source file:org.apache.tez.test.MiniTezCluster.java

License:Apache License

@Override
public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
    // Use libs from cluster since no build is available
    conf.setBoolean(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS, true);
    // blacklisting disabled to prevent scheduling issues
    conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                new File(getTestWorkDir(), "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
    }/*from   w  w w . j av  a2  s  .c o m*/

    if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) {
        // nothing defined. set quick delete value
        conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
    }

    File appJarLocalFile = new File(MiniTezCluster.APPJAR);

    if (!appJarLocalFile.exists()) {
        String message = "TezAppJar " + MiniTezCluster.APPJAR + " not found. Exiting.";
        LOG.info(message);
        throw new TezUncheckedException(message);
    } else {
        LOG.info("Using Tez AppJar: " + appJarLocalFile.getAbsolutePath());
    }

    FileSystem fs = FileSystem.get(conf);
    Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir"));
    Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar");
    // Copy AppJar and make it public.
    Path appMasterJar = new Path(MiniTezCluster.APPJAR);
    fs.copyFromLocalFile(appMasterJar, appRemoteJar);
    fs.setPermission(appRemoteJar, new FsPermission("777"));

    conf.set(TezConfiguration.TEZ_LIB_URIS, appRemoteJar.toUri().toString());
    LOG.info("Set TEZ-LIB-URI to: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

    // VMEM monitoring disabled, PMEM monitoring enabled.
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);

    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
        Path stagingPath = FileContext.getFileContext(conf)
                .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        /*
         * Re-configure the staging path on Windows if the file system is localFs.
         * We need to use a absolute path that contains the drive letter. The unit
         * test could run on a different drive than the AM. We can run into the
         * issue that job files are localized to the drive where the test runs on,
         * while the AM starts on a different drive and fails to find the job
         * metafiles. Using absolute path can avoid this ambiguity.
         */
        if (Path.WINDOWS) {
            if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
                conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                        new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath());
            }
        }
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        fc.mkdir(stagingPath, null, true);

        //mkdir done directory as well
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new TezUncheckedException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test");

    //configure the shuffle service in NM
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
            new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
    conf.setClass(
            String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID),
            ShuffleHandler.class, Service.class);

    // Non-standard shuffle port
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class,
            ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    super.serviceInit(conf);
}