List of usage examples for org.apache.hadoop.hdfs MiniDFSCluster MiniDFSCluster
@Deprecated public MiniDFSCluster(int nameNodePort, Configuration conf, int numDataNodes, boolean format, boolean manageDfsDirs, StartupOption operation, String[] racks) throws IOException
From source file:alluxio.underfs.hdfs.LocalMiniDFSCluster.java
License:Apache License
/** * Starts the minidfscluster before using it. *///from w w w. ja v a 2 s . c o m @Override public void start() throws IOException { if (!mIsStarted) { delete(mBaseDir, true); if (!mkdirs(mBaseDir)) { throw new IOException("Failed to make folder: " + mBaseDir); } // TODO(hy): For hadoop 1.x, there exists NPE while startDataNode. It is a known issue caused // by "umask 002" (should be 022) see [HDFS-2556]. So the following code only works for // hadoop 2.x or "umask 022". System.setProperty("test.build.data", mBaseDir); mDfsCluster = new MiniDFSCluster(mNamenodePort, mConf, mNumDataNode, true, true, null, null); mDfsCluster.waitClusterUp(); if (0 == mNamenodePort) { mNamenodePort = mDfsCluster.getNameNodePort(); } // For HDFS of earlier versions, getFileSystem() returns an instance of type // {@link org.apache.hadoop.fs.FileSystem} rather than {@link DistributedFileSystem} mDfsClient = (DistributedFileSystem) mDfsCluster.getFileSystem(); mIsStarted = true; } }
From source file:com.cloudera.llama.am.MiniLlama.java
License:Apache License
private Configuration startMiniHadoop() throws Exception { int clusterNodes = getConf().getInt(MINI_CLUSTER_NODES_KEY, 1); if (System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA) == null) { String testBuildData = new File("target").getAbsolutePath(); System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testBuildData); }//from w w w . ja v a 2s . co m //to trigger hdfs-site.xml registration as default resource new HdfsConfiguration(); Configuration conf = new YarnConfiguration(); String llamaProxyUser = System.getProperty("user.name"); conf.set("hadoop.security.authentication", "simple"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*"); String[] userGroups = new String[] { "g" }; UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups); int hdfsPort = 0; String fsUri = conf.get("fs.defaultFS"); if (fsUri != null && !fsUri.equals("file:///")) { int i = fsUri.lastIndexOf(":"); if (i > -1) { try { hdfsPort = Integer.parseInt(fsUri.substring(i + 1)); } catch (Exception ex) { throw new RuntimeException( "Could not parse port from Hadoop's " + "'fs.defaultFS property: " + fsUri); } } } miniHdfs = new MiniDFSCluster(hdfsPort, conf, clusterNodes, !skipDfsFormat, true, null, null); miniHdfs.waitActive(); conf = miniHdfs.getConfiguration(0); miniYarn = new MiniYARNCluster("minillama", clusterNodes, 1, 1); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); miniYarn.init(conf); miniYarn.start(); conf = miniYarn.getConfig(); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); return conf; }
From source file:com.dasasian.chok.testutil.integration.HadoopMiniCluster.java
License:Apache License
public void start() throws IOException { LOG.info("starting hadoop cluster..."); Configuration conf = new Configuration(); System.setProperty("test.build.data", "build"); System.setProperty("hadoop.log.dir", "build/logs"); _dfsCluster = new MiniDFSCluster(_namenodePort, conf, _datanodeCount, true, true, StartupOption.REGULAR, null);// ww w . ja v a2s . co m LOG.info("started namnode on " + conf.get(IHadoopConstants.NAMENODE)); _mrCluster = new MiniMRCluster(_jobtrackerPort, 0, _tasktrackerCount, conf.get(IHadoopConstants.NAMENODE), 1); LOG.info("started jobtracker on " + _mrCluster.createJobConf().get(IHadoopConstants.JOBTRACKER)); }
From source file:eu.stratosphere.pact.test.util.filesystem.MiniDFSProvider.java
License:Apache License
public void start() throws Exception { deleteDir(getTempDir());//from w ww . ja va 2 s . c om // create dirs new FileWriter().dir(getTempDir()); new FileWriter().dir(getLogDir()); new FileWriter().dir(getDfsDir()); new FileWriter().dir(getConfigDir()); // set system properies needed by the MiniDFSCluster System.setProperty("hadoop.log.dir", getLogDir()); System.setProperty("test.build.data", getDfsDir()); // init hdfs cluster cluster = new MiniDFSCluster(NAME_NODE_PORT, new Configuration(), 1, true, true, null, null); hdfs = cluster.getFileSystem(); // write hdfs config files needed by nephele in temp folder new FileWriter().dir(getConfigDir()).file("hadoop-site.xml") .write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "<?xml-stylesheet href=\"configuration.xsl\" type=\"text/xsl\"?>", "<configuration>", " <property>", " <name>fs.default.name</name>", " <value>" + "hdfs://localhost:" + cluster.getNameNodePort() + "</value>", " </property>", " <property>", " <name>dfs.name.dir</name>", " <value>" + getDfsDir() + "/name</value>", " </property>", " <property>", " <name>dfs.data.dir</name>", " <value>" + getDfsDir() + "/data</value>", " </property>", "</configuration>") .close().file("hadoop-default.xml") .write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "<?xml-stylesheet href=\"configuration.xsl\" type=\"text/xsl\"?>", "<configuration>", "</configuration>") .close(); }
From source file:org.kaaproject.kaa.server.flume.TestKaaHdfsSink.java
License:Apache License
@BeforeClass public static void setUp() throws IOException { int dataNodes = 1; int port = 29999; JobConf conf = new JobConf(); channel = new MemoryChannel(); conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); conf.set("fs.default.name", "hdfs://localhost:29999"); dfsCluster = new MiniDFSCluster(port, conf, dataNodes, true, true, null, null); fileSystem = dfsCluster.getFileSystem(); fileSystem.delete(new Path("/logs"), true); source = new AvroSource(); sink = new KaaHdfsSink(); logSchemasRootDir = new File("schemas"); if (logSchemasRootDir.exists()) { logSchemasRootDir.delete();//from w ww .ja v a 2 s. co m } prepareSchema(logSchemasRootDir); }
From source file:tachyon.LocalMiniDFSCluster.java
License:Apache License
/** * To start the minidfscluster before using it * /*from ww w .j a v a2 s . c o m*/ * @throws IOException */ @Override public void start() throws IOException { if (!mIsStarted) { delete(mBaseDir, true); if (!mkdirs(mBaseDir, mTachyonConf)) { throw new IOException("Failed to make folder: " + mBaseDir); } // TODO For hadoop 1.x, there exists NPE while startDataNode. It is a known issue caused by // "umask 002"(should be 022) see [HDFS-2556]. So the following codes only work for // hadoop 2.x or "umask 022" System.setProperty("test.build.data", mBaseDir); mDfsCluster = new MiniDFSCluster(mNamenodePort, mConf, mNumDataNode, true, true, null, null); mDfsCluster.waitClusterUp(); if (0 == mNamenodePort) { mNamenodePort = mDfsCluster.getNameNodePort(); } mDfsClient = (DistributedFileSystem) mDfsCluster.getFileSystem(); mIsStarted = true; } }
From source file:tachyon.underfs.hdfs.LocalMiniDFSCluster.java
License:Apache License
/** * To start the minidfscluster before using it * * @throws IOException//from ww w .j ava 2s.c om */ @Override public void start() throws IOException { if (!mIsStarted) { delete(mBaseDir, true); if (!mkdirs(mBaseDir, mTachyonConf)) { throw new IOException("Failed to make folder: " + mBaseDir); } // TODO(hy): For hadoop 1.x, there exists NPE while startDataNode. It is a known issue caused // by "umask 002" (should be 022) see [HDFS-2556]. So the following code only works for // hadoop 2.x or "umask 022". System.setProperty("test.build.data", mBaseDir); mDfsCluster = new MiniDFSCluster(mNamenodePort, mConf, mNumDataNode, true, true, null, null); mDfsCluster.waitClusterUp(); if (0 == mNamenodePort) { mNamenodePort = mDfsCluster.getNameNodePort(); } // For HDFS of earlier versions, getFileSystem() returns an instance of type // {@link org.apache.hadoop.fs.FileSystem} rather than {@link DistributedFileSystem} mDfsClient = (DistributedFileSystem) mDfsCluster.getFileSystem(); mIsStarted = true; } }