List of usage examples for org.apache.hadoop.mapred JobConf get
public String get(String name)
name property, null if no such property exists. From source file:edu.stolaf.cs.wmrserver.streaming.PipeMapper.java
License:Apache License
String getPipeCommand(JobConf job) { String str = job.get("stream.map.streamprocessor"); if (str == null) { return str; }/*from ww w .ja va 2 s .c om*/ try { return URLDecoder.decode(str, "UTF-8"); } catch (UnsupportedEncodingException e) { System.err.println("stream.map.streamprocessor in jobconf not found"); return null; } }
From source file:edu.stolaf.cs.wmrserver.streaming.PipeMapRed.java
License:Apache License
void setStreamJobDetails(JobConf job) { jobLog_ = job.get("stream.jobLog_"); String s = job.get("stream.minRecWrittenToEnableSkip_"); if (s != null) { minRecWrittenToEnableSkip_ = Long.parseLong(s); logprintln("JobConf set minRecWrittenToEnableSkip_ =" + minRecWrittenToEnableSkip_); }// ww w . j a v a 2s . co m taskId_ = StreamUtil.getTaskInfo(job_); }
From source file:edu.stolaf.cs.wmrserver.streaming.PipeMapRed.java
License:Apache License
void addJobConfToEnvironment(JobConf conf, Map<String, String> env) { if (debug_) { logprintln("addJobConfToEnvironment: begin"); }/* w w w . ja v a 2 s . c o m*/ Iterator it = conf.iterator(); while (it.hasNext()) { Map.Entry en = (Map.Entry) it.next(); String name = (String) en.getKey(); //String value = (String)en.getValue(); // does not apply variable expansion String value = conf.get(name); // does variable expansion name = safeEnvVarName(name); envPut(env, name, value); } if (debug_) { logprintln("addJobConfToEnvironment: end"); } }
From source file:edu.stolaf.cs.wmrserver.streaming.PipeReducer.java
License:Apache License
String getPipeCommand(JobConf job) { String str = job.get("stream.reduce.streamprocessor"); if (str == null) { return str; }//from ww w . j a va 2 s . c o m try { return URLDecoder.decode(str, "UTF-8"); } catch (UnsupportedEncodingException e) { System.err.println("stream.reduce.streamprocessor in jobconf not found"); return null; } }
From source file:edu.stolaf.cs.wmrserver.streaming.StreamJob.java
License:Apache License
public static String createJobJar(JobConf conf, List extraFiles, File tmpDir) throws IOException { ArrayList unjarFiles = new ArrayList(); ArrayList packageFiles = new ArrayList(extraFiles); // Runtime code: ship same version of code as self (job submitter code) // usually found in: build/contrib or build/hadoop-<version>-dev-streaming.jar // First try an explicit spec: it's too hard to find our own location in this case: // $HADOOP_HOME/bin/hadoop jar /not/first/on/classpath/custom-hadoop-streaming.jar // where findInClasspath() would find the version of hadoop-streaming.jar in $HADOOP_HOME String runtimeClasses = conf.get("stream.shipped.hadoopstreaming"); // jar or class dir if (runtimeClasses == null) { runtimeClasses = StreamUtil.findInClasspath(StreamJob.class); }//from w w w .j a v a2 s.co m if (runtimeClasses == null) { throw new IOException("runtime classes not found: " + StreamJob.class.getPackage()); } if (StreamUtil.isLocalJobTracker(conf)) { // don't package class files (they might get unpackaged in "." and then // hide the intended CLASSPATH entry) // we still package everything else (so that scripts and executable are found in // Task workdir like distributed Hadoop) } else { if (new File(runtimeClasses).isDirectory()) { packageFiles.add(runtimeClasses); } else { unjarFiles.add(runtimeClasses); } } if (packageFiles.size() + unjarFiles.size() == 0) { return null; } if (tmpDir == null) { String tmp = conf.get("stream.tmpdir", ""); //, "/tmp/${user.name}/" tmpDir = new File(tmp); } // tmpDir=null means OS default tmp dir File jobJar = File.createTempFile("streamjob", ".jar", tmpDir); System.out .println("packageJobJar: " + packageFiles + " " + unjarFiles + " " + jobJar + " tmpDir=" + tmpDir); jobJar.deleteOnExit(); JarBuilder builder = new JarBuilder(); String jobJarName = jobJar.getAbsolutePath(); builder.merge(packageFiles, unjarFiles, jobJarName); return jobJarName; }
From source file:edu.stolaf.cs.wmrserver.streaming.StreamJob.java
License:Apache License
protected static String getTaskEnvironment(JobConf conf) { return conf.get("stream.addenvironment"); }
From source file:edu.stolaf.cs.wmrserver.streaming.StreamUtil.java
License:Apache License
public static TaskId getTaskInfo(JobConf job) { TaskId res = new TaskId(); String id = job.get("mapred.task.id"); if (isLocalJobTracker(job)) { // it uses difft naming res.mapTask = job.getBoolean("mapred.task.is.map", true); res.jobid = "0"; res.taskid = 0;// ww w . j ava 2 s .c o m res.execid = 0; } else { String[] e = id.split("_"); res.mapTask = e[3].equals("m"); res.jobid = e[1] + "_" + e[2]; res.taskid = Integer.parseInt(e[4]); res.execid = Integer.parseInt(e[5]); } return res; }
From source file:edu.ubc.mirrors.holographs.mapreduce.InvokeMethodMapper.java
License:Open Source License
@Override public void configure(final JobConf job) { super.configure(job); snapshot = SnapshotUtils.openSnapshot(job); holographVM = HolographVMRegistry.getHolographVM(snapshot, new VoidProgressListener()); thread = holographVM.getThreads().get(0); Reflection.withThread(thread, new Callable<Void>() { @Override/*from ww w .j a va 2s . c o m*/ public Void call() throws Exception { try { ClassMirror appClassLoaderClass = holographVM .findBootstrapClassMirror("sun.misc.Launcher$AppClassLoader"); InstanceMirror appClassLoader = (InstanceMirror) appClassLoaderClass.getInstances().get(0); ClassMirror urlClassLoaderClass = holographVM .findBootstrapClassMirror(URLClassLoader.class.getName()); InstanceMirror ucp = (InstanceMirror) appClassLoader .get(urlClassLoaderClass.getDeclaredField("ucp")); // ClassMirror urlClassPathClass = holographVM.findBootstrapClassMirror("sun.misc.URLClassPath"); // InstanceMirror stack = (InstanceMirror)ucp.get(urlClassPathClass.getDeclaredField("stack")); } catch (IllegalAccessException e) { // TODO Auto-generated catch block throw new RuntimeException(e); } if (HologramClassLoader.debug) { System.out.println("Finding target class..."); } String targetClassName = job.get("targetClassName"); ClassMirror nameClass = holographVM.findAllClasses(targetClassName, false).get(0); String analyzerClassName = "org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTNameDuplicateAnalysis"; String path = "/Users/robinsalkeld/Documents/UBC/Code/org.eclipse.cdt.git/core/org.eclipse.cdt.core/bin/" + analyzerClassName.replace('.', '/') + ".class"; String classPath = "/Users/robinsalkeld/Documents/UBC/Code/org.eclipse.cdt.git/core/org.eclipse.cdt.core/bin/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPASTNameDuplicateAnalysis.class"; FileInputStream fis = new FileInputStream(path); byte[] analyzerClassBytecode = NativeClassMirror.readFully(fis); fis.close(); ClassMirror analyzerClass = Reflection.injectBytecode(holographVM, thread, nameClass.getLoader(), analyzerClassName, analyzerClassBytecode); method = analyzerClass.getDeclaredMethod("locationKey", targetClassName); return null; } }); }
From source file:edu.ubc.mirrors.holographs.mapreduce.SnapshotUtils.java
License:Open Source License
public static ISnapshot openSnapshot(JobConf job) { // TODO-RS: Accept arbitrary path, copy locally // Path snapshotPath = FileInputFormat.getInputPaths(job)[0]; File snapshotPath = new File(job.get("snapshotPath")); try {//from w w w . j a va 2 s . c o m return SnapshotFactory.openSnapshot(snapshotPath, new VoidProgressListener()); } catch (SnapshotException e) { throw new RuntimeException(e); } }
From source file:edu.ubc.mirrors.holographs.mapreduce.SnapshotUtils.java
License:Open Source License
public static int[] getInputObjectIDs(JobConf job, ISnapshot snapshot) { String targetClassName = job.get("targetClassName"); try {/* w w w. j a va 2 s .c o m*/ IClass klass = snapshot.getClassesByName(targetClassName, false).iterator().next(); return klass.getObjectIds(); } catch (SnapshotException e) { throw new RuntimeException(e); } }