List of usage examples for org.apache.hadoop.fs FileSystem setDefaultUri
public static void setDefaultUri(Configuration conf, String uri)
From source file:cascading.platform.hadoop2.Hadoop2MR1Platform.java
License:Open Source License
@Override public synchronized void setUp() throws IOException { if (configuration != null) return;/* w w w .ja v a 2 s . co m*/ if (!isUseCluster()) { LOG.info("not using cluster"); configuration = new JobConf(); // enforce settings to make local mode behave the same across distributions configuration.set("fs.defaultFS", "file:///"); configuration.set("mapreduce.framework.name", "local"); configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/" + "build/tmp/cascading/staging"); String stagingDir = configuration.get("mapreduce.jobtracker.staging.root.dir"); if (Util.isEmpty(stagingDir)) configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/build/tmp/cascading/staging"); fileSys = FileSystem.get(configuration); } else { LOG.info("using cluster"); if (Util.isEmpty(System.getProperty("hadoop.log.dir"))) System.setProperty("hadoop.log.dir", "build/test/log"); if (Util.isEmpty(System.getProperty("hadoop.tmp.dir"))) System.setProperty("hadoop.tmp.dir", "build/test/tmp"); new File(System.getProperty("hadoop.log.dir")).mkdirs(); // ignored JobConf conf = new JobConf(); if (!Util.isEmpty(System.getProperty("mapred.jar"))) { LOG.info("using a remote cluster with jar: {}", System.getProperty("mapred.jar")); configuration = conf; ((JobConf) configuration).setJar(System.getProperty("mapred.jar")); if (!Util.isEmpty(System.getProperty("fs.default.name"))) { LOG.info("using {}={}", "fs.default.name", System.getProperty("fs.default.name")); configuration.set("fs.default.name", System.getProperty("fs.default.name")); } if (!Util.isEmpty(System.getProperty("mapred.job.tracker"))) { LOG.info("using {}={}", "mapred.job.tracker", System.getProperty("mapred.job.tracker")); configuration.set("mapred.job.tracker", System.getProperty("mapred.job.tracker")); } if (!Util.isEmpty(System.getProperty("fs.defaultFS"))) { LOG.info("using {}={}", "fs.defaultFS", System.getProperty("fs.defaultFS")); configuration.set("fs.defaultFS", System.getProperty("fs.defaultFS")); } if (!Util.isEmpty(System.getProperty("yarn.resourcemanager.address"))) { LOG.info("using {}={}", "yarn.resourcemanager.address", System.getProperty("yarn.resourcemanager.address")); configuration.set("yarn.resourcemanager.address", System.getProperty("yarn.resourcemanager.address")); } if (!Util.isEmpty(System.getProperty("mapreduce.jobhistory.address"))) { LOG.info("using {}={}", "mapreduce.jobhistory.address", System.getProperty("mapreduce.jobhistory.address")); configuration.set("mapreduce.jobhistory.address", System.getProperty("mapreduce.jobhistory.address")); } configuration.set("mapreduce.user.classpath.first", "true"); // use test dependencies configuration.set("mapreduce.framework.name", "yarn"); fileSys = FileSystem.get(configuration); } else { conf.setBoolean("yarn.is.minicluster", true); // conf.setInt( "yarn.nodemanager.delete.debug-delay-sec", -1 ); // conf.set( "yarn.scheduler.capacity.root.queues", "default" ); // conf.set( "yarn.scheduler.capacity.root.default.capacity", "100" ); // disable blacklisting hosts not to fail localhost during unit tests conf.setBoolean("yarn.app.mapreduce.am.job.node-blacklisting.enable", false); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); FileSystem.setDefaultUri(conf, fileSys.getUri()); mr = MiniMRClientClusterFactory.create(this.getClass(), 4, conf); configuration = mr.getConfig(); } configuration.set("mapred.child.java.opts", "-Xmx512m"); configuration.setInt("mapreduce.job.jvm.numtasks", -1); configuration.setInt("mapreduce.client.completion.pollinterval", 50); configuration.setInt("mapreduce.client.progressmonitor.pollinterval", 50); configuration.setBoolean("mapreduce.map.speculative", false); configuration.setBoolean("mapreduce.reduce.speculative", false); } configuration.setInt("mapreduce.job.maps", numMappers); configuration.setInt("mapreduce.job.reduces", numReducers); Map<Object, Object> globalProperties = getGlobalProperties(); if (logger != null) globalProperties.put("log4j.logger", logger); FlowProps.setJobPollingInterval(globalProperties, 10); // should speed up tests Hadoop2MR1Planner.copyProperties(configuration, globalProperties); // copy any external properties Hadoop2MR1Planner.copyConfiguration(properties, configuration); // put all properties on the jobconf }
From source file:com.google.mr4c.hadoop.yarn.YarnTestBinding.java
License:Open Source License
private void startMrCluster() throws IOException { Configuration conf = new JobConf(); FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri()); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); String addr = MiniYARNCluster.getHostname() + ":0"; conf.set(YarnConfiguration.RM_ADDRESS, addr); conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr); m_mrCluster = MiniMRClientClusterFactory.create(HadoopTestUtils.class, "MR4CTests", 1, // num node managers conf);/* w w w . j a va2 s .c o m*/ // make sure startup is finished for (int i = 0; i < 60; i++) { String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); if (newAddr.equals(addr)) { s_log.warn("MiniYARNCluster startup not complete"); try { Thread.sleep(1000); } catch (InterruptedException ie) { throw new IOException(ie); } } else { s_log.info("MiniYARNCluster now available at {}", newAddr); return; } } throw new IOException("MiniYARNCluster taking too long to startup"); }
From source file:common.NameNode.java
License:Apache License
protected void setRpcServerAddress(Configuration conf) { FileSystem.setDefaultUri(conf, getUri(rpcAddress)); }
From source file:ml.shifu.guagua.hadoop.io.GuaguaOptionsParser.java
License:Apache License
/** * Modify configuration according user-specified generic options * /*from w ww . java 2s . c o m*/ * @param conf * Configuration to be modified * @param line * User-specified generic options */ private void processGeneralOptions(Configuration conf, CommandLine line) throws IOException { if (line.hasOption("fs")) { FileSystem.setDefaultUri(conf, line.getOptionValue("fs")); } if (line.hasOption("jt")) { conf.set("mapred.job.tracker", line.getOptionValue("jt")); } if (line.hasOption("conf")) { String[] values = line.getOptionValues("conf"); for (String value : values) { conf.addResource(new Path(value)); } } if (line.hasOption("libjars")) { conf.set("tmpjars", validateFiles(line.getOptionValue("libjars"), conf)); // setting libjars in client classpath URL[] libjars = getLibJars(conf); if (libjars != null && libjars.length > 0) { conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader())); Thread.currentThread().setContextClassLoader( new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader())); } } if (line.hasOption("files")) { conf.set("tmpfiles", validateFiles(line.getOptionValue("files"), conf)); } if (line.hasOption("archives")) { conf.set("tmparchives", validateFiles(line.getOptionValue("archives"), conf)); } if (line.hasOption('D')) { String[] property = line.getOptionValues('D'); for (String prop : property) { String[] keyval = prop.split("=", 2); if (keyval.length == 2) { conf.set(keyval[0], keyval[1]); } } } conf.setBoolean("mapred.used.genericoptionsparser", true); // tokensFile if (line.hasOption("tokenCacheFile")) { String fileName = line.getOptionValue("tokenCacheFile"); // check if the local file exists try { FileSystem localFs = FileSystem.getLocal(conf); Path p = new Path(fileName); if (!localFs.exists(p)) { throw new FileNotFoundException("File " + fileName + " does not exist."); } LOG.debug("setting conf tokensFile: {}", fileName); conf.set("mapreduce.job.credentials.json", localFs.makeQualified(p).toString()); } catch (IOException e) { throw new RuntimeException(e); } } }
From source file:org.apache.ambari.servicemonitor.clients.BaseClient.java
License:Apache License
private void processBaseOptions(Configuration conf, CommandLine commandLine) { if (commandLine.hasOption('v')) { out.setVerbose(true);/*from w w w.j a v a 2 s.c o m*/ } if (commandLine.hasOption('c')) { out.setColorsSet(true); } if (commandLine.hasOption('b')) { LOG.info("blocking"); blockingIO = true; } else { LOG.info("non-blocking"); } if (commandLine.hasOption('h')) { keepHistory = true; } attemptLimit = getIntOption(commandLine, "al"); int processTimeout = getIntOption(commandLine, "processtimeout"); if (processTimeout > 0) { killHungProcess = new KillHungProcess(processTimeout, "Timeout executing process"); } failLimit = getIntOption(commandLine, "fl"); successLimit = getIntOption(commandLine, "sl"); sleepTime = getIntOption(commandLine, "st"); if (sleepTime < 0) { sleepTime = DEFAULT_SLEEP; } if (commandLine.hasOption("fs")) { FileSystem.setDefaultUri(conf, commandLine.getOptionValue("fs")); } if (commandLine.hasOption("jt")) { conf.set("mapred.job.tracker", commandLine.getOptionValue("jt")); } if (commandLine.hasOption("conf")) { String[] values = commandLine.getOptionValues("conf"); for (String value : values) { conf.addResource(new Path(value)); } } }
From source file:org.apache.blur.mapreduce.lib.BlurOutputFormatMiniClusterTest.java
License:Apache License
@BeforeClass public static void setupTest() throws Exception { GCWatcher.init(0.60);//from ww w .j ava 2 s . co m JavaHome.checkJavaHome(); LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, false); TEST_ROOT_DIR = new Path(miniCluster.getFileSystemUri().toString() + "/blur_test"); System.setProperty("hadoop.log.dir", "./target/BlurOutputFormatTest/hadoop_log"); try { fileSystem = TEST_ROOT_DIR.getFileSystem(conf); } catch (IOException io) { throw new RuntimeException("problem getting local fs", io); } FileSystem.setDefaultUri(conf, miniCluster.getFileSystemUri()); miniCluster.startMrMiniCluster(); conf = miniCluster.getMRConfiguration(); BufferStore.initNewBuffer(128, 128 * 128); }
From source file:org.apache.hoya.yarn.params.ArgOps.java
License:Apache License
public static void applyFileSystemURL(URI filesystemURL, Configuration conf) { if (filesystemURL != null) { //filesystem argument was set -this overwrites any defaults in the //configuration FileSystem.setDefaultUri(conf, filesystemURL); }//from www . j av a 2s . c o m }
From source file:org.apache.pig.piggybank.test.storage.TestAllLoader.java
License:Apache License
@Before public void setUp() throws Exception { Configuration hadoopConf = new Configuration(); FileSystem.setDefaultUri(hadoopConf, LocalFileSystem.getDefaultUri(hadoopConf)); if (baseDir == null) { configuration = new Properties(); configuration.setProperty(LoadFuncHelper.FILE_EXTENSION_LOADERS, extensionLoaders); baseDir = new File("build/test/testAllLoader"); if (baseDir.exists()) { FileUtil.fullyDelete(baseDir); }//from w w w .jav a 2 s . com assertTrue(baseDir.mkdirs()); createSimpleDir(); createDatePartitionDir(); createLogicPartitionDir(); createTaggedLogicPartitionDir(); createFileByContentDir(); server = new PigServer(ExecType.LOCAL, configuration); server.setBatchOn(); } }
From source file:org.apache.slider.common.params.ArgOps.java
License:Apache License
public static void applyFileSystemBinding(String filesystemBinding, Configuration conf) { if (filesystemBinding != null) { //filesystem argument was set -this overwrites any defaults in the //configuration FileSystem.setDefaultUri(conf, filesystemBinding); }/*from www .java 2s. co m*/ }
From source file:org.apache.solr.hadoop.hack.MiniMRCluster.java
License:Apache License
public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, int numTaskTrackers, String namenode, int numDir, String[] racks, String[] hosts, UserGroupInformation ugi, JobConf conf, int numTrackerToExclude, Clock clock) throws Exception { if (conf == null) conf = new JobConf(); FileSystem.setDefaultUri(conf, namenode); String identifier = this.getClass().getSimpleName() + "_" + Integer.toString(LuceneTestCase.random().nextInt(Integer.MAX_VALUE)); mrClientCluster = MiniMRClientClusterFactory.create(this.getClass(), identifier, numTaskTrackers, conf, new File(conf.get("testWorkDir"))); }