List of usage examples for org.apache.hadoop.mapred JobConf get
public String get(String name, String defaultValue)
name. From source file:com.ebay.erl.mobius.core.mapred.MobiusMultiInputs.java
License:Apache License
public static void addInputPath(JobConf conf, Path anInput, Class<? extends InputFormat> inputFormatClass, Class<? extends AbstractMobiusMapper> mapperClass, byte datasetID, FileSystem fs) throws IOException { MultipleInputs.addInputPath(conf, anInput, inputFormatClass, mapperClass); // override the {@link InputFormat} class set by the {@link MultipleInputs} // as Mobius need to set the set the current dataset id per input split. conf.setInputFormat(MobiusDelegatingInputFormat.class); // MobiusDelegatingInputFormat extends DelegatingInputFormat, which always // call the FileInpupt#setInputs within DelegatingInputFormat#getInputs // regardless of the actual type of <code>inputFormatClass</code>. ///////////////////////////////////////////////////// // start to build the path to dataset ID mapping ///////////////////////////////////////////////////// MultiInputsHelper helper = MultiInputsHelpersRepository.getInstance(conf).getHelper(inputFormatClass); URI uri = helper.getUniquePathByInputFormat(conf, anInput); String aPath = uri.toString(); if (aPath.indexOf(";") >= 0) throw new IllegalArgumentException(aPath + " cannot contains semicolon"); // set the input path to datasetID mapping in the Hadoop configuration. if (conf.get(ConfigureConstants.INPUT_TO_DATASET_MAPPING, "").isEmpty()) { conf.set(ConfigureConstants.INPUT_TO_DATASET_MAPPING, datasetID + ";" + aPath); } else {/*from w w w. jav a2 s .c om*/ String previous = conf.get(ConfigureConstants.INPUT_TO_DATASET_MAPPING); conf.set(ConfigureConstants.INPUT_TO_DATASET_MAPPING, datasetID + ";" + aPath + "," + previous); } //LOGGER.debug(conf.get(ConfigureConstants.INPUT_TO_DATASET_MAPPING, "")); }
From source file:com.ebay.erl.mobius.core.mapred.MultiInputsHelpersRepository.java
License:Apache License
/** * constructor/*from w w w .jav a 2 s .c o m*/ */ private MultiInputsHelpersRepository(JobConf conf) { this.mapping = new TreeMap<Class<? extends InputFormat>, MultiInputsHelper>(new ClassComparator()); this.register(FileInputFormat.class, FileInputFormatHelper.class); if (!conf.get("mobius.multi.inputs.helpers", "").isEmpty()) { // mobius.multi.inputs.helpers in the format of> InputFormatClassName:HelperClassName(,InputFormatClassName:HelperClassName)? String[] helpers = conf.getStrings("mobius.multi.inputs.helpers"); for (String aHeler : helpers) { String[] data = aHeler.split(":"); String inputFormatClassName = data[0]; String helperClassName = data[1]; Class<? extends InputFormat> inputFormat = (Class<? extends InputFormat>) Util .getClass(inputFormatClassName); Class<? extends MultiInputsHelper> helperClass = (Class<? extends MultiInputsHelper>) Util .getClass(helperClassName); this.register(inputFormat, helperClass); } } }
From source file:com.ebay.erl.mobius.core.mapred.MultiInputsHelpersRepository.java
License:Apache License
public void writeToConf(JobConf conf) { Iterator<Entry<Class<? extends InputFormat>, MultiInputsHelper>> entries = this.mapping.entrySet() .iterator();/* w w w .jav a2 s .c o m*/ while (entries.hasNext()) { Entry<Class<? extends InputFormat>, MultiInputsHelper> anEntry = entries.next(); Class<? extends InputFormat> inputFormat = anEntry.getKey(); Class<? extends MultiInputsHelper> helper = anEntry.getValue().getClass(); if (!conf.get("mobius.multi.inputs.helpers", "").isEmpty()) { String others = conf.get("mobius.multi.inputs.helpers"); conf.set("mobius.multi.inputs.helpers", others + "," + inputFormat.getCanonicalName() + ":" + helper.getCanonicalName()); } else { conf.set("mobius.multi.inputs.helpers", inputFormat.getCanonicalName() + ":" + helper.getCanonicalName()); } } }
From source file:com.ibm.bi.dml.api.DMLScript.java
License:Open Source License
/** * // w w w .j ava2 s . com * @param config * @throws IOException * @throws DMLRuntimeException */ private static void checkSecuritySetup(DMLConfig config) throws IOException, DMLRuntimeException { //analyze local configuration String userName = System.getProperty("user.name"); HashSet<String> groupNames = new HashSet<String>(); try { //check existence, for backwards compatibility to < hadoop 0.21 if (UserGroupInformation.class.getMethod("getCurrentUser") != null) { String[] groups = UserGroupInformation.getCurrentUser().getGroupNames(); for (String g : groups) groupNames.add(g); } } catch (Exception ex) { } //analyze hadoop configuration JobConf job = ConfigurationManager.getCachedJobConf(); boolean localMode = InfrastructureAnalyzer.isLocalMode(job); String taskController = job.get("mapred.task.tracker.task-controller", "org.apache.hadoop.mapred.DefaultTaskController"); String ttGroupName = job.get("mapreduce.tasktracker.group", "null"); String perm = job.get(MRConfigurationNames.DFS_PERMISSIONS, "null"); //note: job.get("dfs.permissions.supergroup",null); URI fsURI = FileSystem.getDefaultUri(job); //determine security states boolean flagDiffUser = !(taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user || localMode // run in the same JVM anyway || groupNames.contains(ttGroupName)); //user in task tracker group boolean flagLocalFS = fsURI == null || fsURI.getScheme().equals("file"); boolean flagSecurity = perm.equals("yes"); LOG.debug("SystemML security check: " + "local.user.name = " + userName + ", " + "local.user.groups = " + ProgramConverter.serializeStringCollection(groupNames) + ", " + "mapred.job.tracker = " + job.get("mapred.job.tracker") + ", " + "mapred.task.tracker.task-controller = " + taskController + "," + "mapreduce.tasktracker.group = " + ttGroupName + ", " + "fs.default.name = " + ((fsURI != null) ? fsURI.getScheme() : "null") + ", " + MRConfigurationNames.DFS_PERMISSIONS + " = " + perm); //print warning if permission issues possible if (flagDiffUser && (flagLocalFS || flagSecurity)) { LOG.warn("Cannot run map/reduce tasks as user '" + userName + "'. Using tasktracker group '" + ttGroupName + "'."); } //validate external filenames working directories String localtmpdir = config.getTextValue(DMLConfig.LOCAL_TMP_DIR); String hdfstmpdir = config.getTextValue(DMLConfig.SCRATCH_SPACE); if (!LocalFileUtils.validateExternalFilename(localtmpdir, false)) throw new DMLRuntimeException("Invalid (non-trustworthy) local working directory."); if (!LocalFileUtils.validateExternalFilename(hdfstmpdir, true)) throw new DMLRuntimeException("Invalid (non-trustworthy) hdfs working directory."); }
From source file:com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer.java
License:Open Source License
public static boolean isLocalMode(JobConf job) { // Due to a bug in HDP related to fetching the "mode" of execution within mappers, // we explicitly probe the relevant properties instead of relying on results from // analyzeHadoopCluster(). String jobTracker = job.get("mapred.job.tracker", "local"); String framework = job.get("mapreduce.framework.name", "local"); boolean isYarnEnabled = (framework != null && framework.equals("yarn")); return ("local".equals(jobTracker) & !isYarnEnabled); }
From source file:com.ibm.bi.dml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer.java
License:Open Source License
/** * //from w w w .j a va 2 s . c o m * @param job * @return */ private static boolean analyzeLocalMode(JobConf job) { //analyze if local mode (if yarn enabled, we always assume cluster mode //in order to workaround configuration issues on >=Hadoop 2.6) String jobTracker = job.get("mapred.job.tracker", "local"); return "local".equals(jobTracker) & !isYarnEnabled(); }
From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java
License:Open Source License
/** * Unique working dirs required for thread-safe submission of parallel jobs; * otherwise job.xml and other files might be overridden (in local mode). * /*from ww w . j a va2 s .com*/ * @param job * @param mode */ public static void setUniqueWorkingDir(JobConf job) { if (InfrastructureAnalyzer.isLocalMode(job)) { StringBuilder tmp = new StringBuilder(); tmp.append(Lop.FILE_SEPARATOR); tmp.append(Lop.PROCESS_PREFIX); tmp.append(DMLScript.getUUID()); tmp.append(Lop.FILE_SEPARATOR); tmp.append(seq.getNextID()); String uniqueSubdir = tmp.toString(); //unique local dir String[] dirlist = job.get("mapred.local.dir", "/tmp").split(","); StringBuilder sb2 = new StringBuilder(); for (String dir : dirlist) { if (sb2.length() > 0) sb2.append(","); sb2.append(dir); sb2.append(uniqueSubdir); } job.set("mapred.local.dir", sb2.toString()); //unique system dir job.set("mapred.system.dir", job.get("mapred.system.dir") + uniqueSubdir); //unique staging dir job.set("mapreduce.jobtracker.staging.root.dir", job.get("mapreduce.jobtracker.staging.root.dir") + uniqueSubdir); } }
From source file:com.ibm.bi.dml.runtime.matrix.sort.ValueSortMapper.java
License:Open Source License
@Override @SuppressWarnings("unchecked") public void configure(JobConf job) { try {/*from w w w . j ava 2s .co m*/ brlen = MRJobConfiguration.getNumRowsPerBlock(job, (byte) 0); bclen = MRJobConfiguration.getNumColumnsPerBlock(job, (byte) 0); String str = job.get(SortMR.COMBINE_INSTRUCTION, null); if (str != null && !str.isEmpty() && !"null".equals(str)) combineInstruction = (CombineUnaryInstruction) CombineUnaryInstruction.parseInstruction(str); inputConverter = MRJobConfiguration.getInputConverter(job, (byte) 0); inputConverter.setBlockSize(brlen, bclen); } catch (DMLRuntimeException e) { throw new RuntimeException(e); } }
From source file:com.ibm.jaql.io.hadoop.CompositeInputAdapter.java
License:Apache License
/** * @param job/* w w w . j av a2 s . com*/ * @return */ public static int readCurrentIndex(JobConf job) { String v = job.get(CURRENT_IDX_NAME, "0"); return Integer.parseInt(v); }
From source file:com.linkedin.haivvreo.AvroContainerOutputFormat.java
License:Apache License
@Override public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException { Schema schema;//from w ww .ja va 2s . c om try { schema = HaivvreoUtils.determineSchemaOrThrowException(properties); } catch (HaivvreoException e) { throw new IOException(e); } GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema); DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw); if (isCompressed) { int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC); CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); dfw.setCodec(factory); } dfw.create(schema, path.getFileSystem(jobConf).create(path)); return new AvroGenericRecordWriter(dfw); }