Example usage for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR

List of usage examples for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.hdfs.*
  5. MiniDFSCluster
  6. HDFS_MINIDFS_BASEDIR

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR.

Prototype

String HDFS_MINIDFS_BASEDIR

To view the source code for org.apache.hadoop.hdfs MiniDFSCluster HDFS_MINIDFS_BASEDIR.

Click Source Link

Document

Configuration option to set the data dir:

Usage

From source file:backup.integration.MiniClusterTestBase.java

License:Apache License

private Configuration setupConfig(File hdfsDir) throws Exception {
    Configuration conf = new Configuration();
    File backup = new File(tmpHdfs, "backup");
    backup.mkdirs();/*from  w  w  w .j av a  2 s  . c o m*/
    conf.set(DFS_BACKUP_NAMENODE_LOCAL_DIR_KEY, backup.getAbsolutePath());
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
    conf.set(DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY, BackupFsDatasetSpiFactory.class.getName());
    conf.set(DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY, DataNodeBackupServicePlugin.class.getName());
    conf.set(DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY, NameNodeBackupServicePlugin.class.getName());

    conf.setInt(BackupConstants.DFS_BACKUP_DATANODE_RPC_PORT_KEY, 0);
    conf.setInt(BackupConstants.DFS_BACKUP_NAMENODE_HTTP_PORT_KEY, 0);

    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 2);// 3
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_KEY, 2);// 3
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 6000);// 30000
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 6000);// 5*60*1000

    org.apache.commons.configuration.Configuration configuration = BackupUtil.convert(conf);
    setupBackupStore(configuration);
    @SuppressWarnings("unchecked")
    Iterator<String> keys = configuration.getKeys();
    while (keys.hasNext()) {
        String key = keys.next();
        conf.set(key, configuration.getString(key));
    }

    return conf;
}

From source file:cascading.platform.tez.Hadoop2TezPlatform.java

License:Open Source License

@Override
public synchronized void setUp() throws IOException {
    if (configuration != null)
        return;//from   w ww  .  j ava 2  s  .  com

    if (!isUseCluster()) {
        // Current usage requirements:
        // 1. Clients need to set "tez.local.mode" to true when creating a TezClient instance. (For the examples this can be done via -Dtez.local.mode=true)
        // 2. fs.defaultFS must be set to "file:///"
        // 2.1 If running examples - this must be set in tez-site.xml (so that it's picked up by the client, as well as the conf instances used to configure the Inputs / Outputs).
        // 2.2 If using programatically (without a tez-site.xml present). All configuration instances used (to crate the client / configure Inputs / Outputs) - must have this property set.
        // 3. tez.runtime.optimize.local.fetch needs to be set to true (either via tez-site.xml or in all configurations used to create the job (similar to fs.defaultFS in step 2))
        // 4. tez.staging-dir must be set (either programatically or via tez-site.xml).
        // Until TEZ-1337 goes in - the staging-dir for the job is effectively the root of the filesystem (and where inputs are read from / written to if relative paths are used).

        LOG.info("not using cluster");
        configuration = new Configuration();

        configuration.setInt(FlowRuntimeProps.GATHER_PARTITIONS, getNumGatherPartitions());
        //      configuration.setInt( FlowRuntimeProps.GATHER_PARTITIONS, 1 ); // deadlocks if larger than 1

        configuration.set(TezConfiguration.TEZ_LOCAL_MODE, "true");
        configuration.set("fs.defaultFS", "file:///");
        configuration.set("tez.runtime.optimize.local.fetch", "true");

        // hack to prevent deadlocks where downstream processors are scheduled before upstream
        configuration.setInt("tez.am.inline.task.execution.max-tasks", 3); // testHashJoinMergeIntoHashJoinAccumulatedAccumulatedMerge fails if set to 2

        configuration.set(TezConfiguration.TEZ_IGNORE_LIB_URIS, "true"); // in local mode, use local classpath
        configuration.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, -1);
        configuration.set(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS, "true");

        configuration.set("tez.am.mode.session", "true"); // allows multiple TezClient instances to be used in a single jvm

        if (!Util.isEmpty(System.getProperty("hadoop.tmp.dir")))
            configuration.set("hadoop.tmp.dir", System.getProperty("hadoop.tmp.dir"));
        else
            configuration.set("hadoop.tmp.dir", "build/test/tmp");

        fileSys = FileSystem.get(configuration);
    } else {
        LOG.info("using cluster");

        if (Util.isEmpty(System.getProperty("hadoop.log.dir")))
            System.setProperty("hadoop.log.dir", "build/test/log");

        if (Util.isEmpty(System.getProperty("hadoop.tmp.dir")))
            System.setProperty("hadoop.tmp.dir", "build/test/tmp");

        new File(System.getProperty("hadoop.log.dir")).mkdirs(); // ignored
        new File(System.getProperty("hadoop.tmp.dir")).mkdirs(); // ignored

        Configuration defaultConf = new Configuration();

        defaultConf.setInt(FlowRuntimeProps.GATHER_PARTITIONS, getNumGatherPartitions());

        defaultConf.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, -1);

        //      defaultConf.set( TezConfiguration.TEZ_AM_LOG_LEVEL, "DEBUG" );
        //      defaultConf.set( TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG" );

        defaultConf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
        defaultConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
        defaultConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, System.getProperty("hadoop.tmp.dir"));

        miniDFSCluster = new MiniDFSCluster.Builder(defaultConf).numDataNodes(4).format(true).racks(null)
                .build();

        fileSys = miniDFSCluster.getFileSystem();

        Configuration tezConf = new Configuration(defaultConf);
        tezConf.set("fs.defaultFS", fileSys.getUri().toString()); // use HDFS
        tezConf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");

        // see MiniTezClusterWithTimeline as alternate
        miniTezCluster = new MiniTezCluster(getClass().getName(), 4, 1, 1); // todo: set to 4
        miniTezCluster.init(tezConf);
        miniTezCluster.start();

        configuration = miniTezCluster.getConfig();

        // stats won't work after completion unless ATS is used
        if (setTimelineStore(configuration)) // true if ats can be loaded and configured for this hadoop version
        {
            configuration.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
                    ATSHistoryLoggingService.class.getName());
            configuration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
            configuration.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, "localhost:10200");
            configuration.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:8188");
            configuration.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, "localhost:8190");

            yarnHistoryServer = new ApplicationHistoryServer();
            yarnHistoryServer.init(configuration);
            yarnHistoryServer.start();
        }
    }

    configuration.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 1);
    configuration.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    configuration.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 1);

    Map<Object, Object> globalProperties = getGlobalProperties();

    if (logger != null)
        globalProperties.put("log4j.logger", logger);

    FlowProps.setJobPollingInterval(globalProperties, 10); // should speed up tests

    Hadoop2TezPlanner.copyProperties(configuration, globalProperties); // copy any external properties

    Hadoop2TezPlanner.copyConfiguration(properties, configuration); // put all properties on the jobconf

    ExitUtil.disableSystemExit();

    //    forbidSystemExitCall();
}

From source file:co.cask.cdap.app.runtime.spark.distributed.SparkExecutionServiceTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();/*  ww w .  j  a  v a 2s .  co m*/
    locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
}

From source file:co.cask.cdap.common.io.DFSSeekableInputStreamTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();//from  ww  w  .  ja v a2 s  .  co m
    locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
}

From source file:co.cask.cdap.data.stream.DFSStreamFileJanitorTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {

    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();//from w  w w . j av a  2 s .c  om
    final LocationFactory lf = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    final NamespacedLocationFactory nlf = new DefaultNamespacedLocationFactory(cConf, lf);

    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(),
            new AbstractModule() {
                @Override
                protected void configure() {
                    bind(LocationFactory.class).toInstance(lf);
                    bind(NamespacedLocationFactory.class).toInstance(nlf);
                }
            }, new TransactionMetricsModule(), new DiscoveryRuntimeModule().getInMemoryModules(),
            new DataFabricModules().getDistributedModules(),
            Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(MetadataStore.class).to(NoOpMetadataStore.class);
                }
            }), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(),
            Modules.override(new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

                @Override
                protected void configure() {
                    // Tests are running in same process, hence no need to have ZK to coordinate
                    bind(StreamCoordinatorClient.class).to(InMemoryStreamCoordinatorClient.class)
                            .in(Scopes.SINGLETON);
                    bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
                }
            }), new AbstractModule() {
                @Override
                protected void configure() {
                    // We don't need notification in this test, hence inject an no-op one
                    bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
                    bind(NamespaceStore.class).to(InMemoryNamespaceStore.class);
                }
            });

    locationFactory = injector.getInstance(LocationFactory.class);
    namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
    namespaceStore = injector.getInstance(NamespaceStore.class);
    streamAdmin = injector.getInstance(StreamAdmin.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    streamCoordinatorClient.startAndWait();
}

From source file:co.cask.cdap.data.stream.DFSTimePartitionedStreamTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();//from  ww  w .j  a  v a  2  s. c  o m
    locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
}

From source file:co.cask.cdap.data.stream.DistributedStreamCoordinatorClientTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();/*from ww  w  . j  a v a2  s  .  c  om*/

    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();
    final LocationFactory lf = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    final NamespacedLocationFactory nlf = new DefaultNamespacedLocationFactory(cConf, lf);

    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());

    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(),
            new DiscoveryRuntimeModule().getDistributedModules(),
            new DataFabricModules().getDistributedModules(),
            Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(MetadataStore.class).to(NoOpMetadataStore.class);
                }
            }), new TransactionMetricsModule(), new NotificationFeedServiceRuntimeModule().getInMemoryModules(),
            new AbstractModule() {
                @Override
                protected void configure() {
                    bind(LocationFactory.class).toInstance(lf);
                    bind(NamespacedLocationFactory.class).toInstance(nlf);
                }
            }, new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(),
            Modules.override(new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
                }
            }));

    zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();

    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    streamAdmin = injector.getInstance(StreamAdmin.class);
    coordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    coordinatorClient.startAndWait();
}

From source file:co.cask.cdap.data.stream.service.DFSConcurrentStreamWriterTest.java

License:Apache License

@BeforeClass
public static void init() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    dfsCluster.waitClusterUp();//  ww  w. j av  a 2s .c  o m
    LocationFactory locationFactory = new FileContextLocationFactory(dfsCluster.getFileSystem().getConf());
    namespacedLocationFactory = new DefaultNamespacedLocationFactory(CConfiguration.create(), locationFactory);

}

From source file:co.cask.cdap.operations.hdfs.HDFSOperationalStatsTest.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath());
    dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(2).build();
    dfsCluster.waitClusterUp();/*from  w w w. j  a  v  a2  s .  c  om*/
}

From source file:co.cask.cdap.security.impersonation.UGIProviderTest.java

License:Apache License

@BeforeClass
public static void init() throws Exception {
    cConf = CConfiguration.create();// w ww . j  av  a  2 s  .  c o  m
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());

    // Start KDC
    miniKdc = new MiniKdc(MiniKdc.createConf(), TEMP_FOLDER.newFolder());
    miniKdc.start();
    System.setProperty("java.security.krb5.conf", miniKdc.getKrb5conf().getAbsolutePath());

    // Generate keytab
    keytabFile = TEMP_FOLDER.newFile();
    miniKdc.createPrincipal(keytabFile, "hdfs", "alice", "bob");

    // Start mini DFS cluster
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    hConf.setBoolean("ipc.client.fallback-to-simple-auth-allowed", true);

    miniDFSCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    miniDFSCluster.waitClusterUp();
    locationFactory = new FileContextLocationFactory(miniDFSCluster.getFileSystem().getConf());

    hConf = new Configuration();
    hConf.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(hConf);
}