List of usage examples for org.apache.hadoop.mapred JobConf getRaw
public String getRaw(String name)
name property, without doing variable expansion.If the key is deprecated, it returns the value of the first key which replaces the deprecated key and is not null. From source file:cascading.flow.hadoop.FlowMapper.java
License:Open Source License
@Override public void configure(JobConf jobConf) { try {// w w w .java 2s .com HadoopUtil.initLog4j(jobConf); LOG.info("cascading version: {}", jobConf.get("cascading.version", "")); LOG.info("child jvm opts: {}", jobConf.get("mapred.child.java.opts", "")); currentProcess = new HadoopFlowProcess(new FlowSession(), jobConf, true); String mapNodeState = jobConf.getRaw("cascading.flow.step.node.map"); if (mapNodeState == null) mapNodeState = readStateFromDistCache(jobConf, jobConf.get(FlowStep.CASCADING_FLOW_STEP_ID), "map"); flowNode = deserializeBase64(mapNodeState, jobConf, BaseFlowNode.class); LOG.info("flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal()); Tap source = Flows.getTapForID(flowNode.getSourceTaps(), jobConf.get("cascading.step.source")); streamGraph = new HadoopMapStreamGraph(currentProcess, flowNode, source); for (Duct head : streamGraph.getHeads()) LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement()); for (Duct tail : streamGraph.getTails()) LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement()); for (Tap trap : flowNode.getTraps()) LOG.info("trapping to: " + trap); logMemory(LOG, "flow node id: " + flowNode.getID() + ", mem on start"); } catch (Throwable throwable) { reportIfLocal(throwable); if (throwable instanceof CascadingException) throw (CascadingException) throwable; throw new FlowException("internal error during mapper configuration", throwable); } }
From source file:cascading.flow.hadoop.FlowReducer.java
License:Open Source License
@Override public void configure(JobConf jobConf) { try {/*from www. ja v a 2 s. c o m*/ super.configure(jobConf); HadoopUtil.initLog4j(jobConf); LOG.info("cascading version: {}", jobConf.get("cascading.version", "")); LOG.info("child jvm opts: {}", jobConf.get("mapred.child.java.opts", "")); currentProcess = new HadoopFlowProcess(new FlowSession(), jobConf, false); timedIterators = TimedIterator.iterators(new TimedIterator<Tuple>(currentProcess, SliceCounters.Read_Duration, SliceCounters.Tuples_Read)); String reduceNodeState = jobConf.getRaw("cascading.flow.step.node.reduce"); if (reduceNodeState == null) reduceNodeState = readStateFromDistCache(jobConf, jobConf.get(FlowStep.CASCADING_FLOW_STEP_ID), "reduce"); flowNode = deserializeBase64(reduceNodeState, jobConf, BaseFlowNode.class); LOG.info("flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal()); streamGraph = new HadoopReduceStreamGraph(currentProcess, flowNode, Util.getFirst(flowNode.getSourceElements())); group = (HadoopGroupGate) streamGraph.getHeads().iterator().next(); for (Duct head : streamGraph.getHeads()) LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement()); for (Duct tail : streamGraph.getTails()) LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement()); for (Tap trap : flowNode.getTraps()) LOG.info("trapping to: " + trap); logMemory(LOG, "flow node id: " + flowNode.getID() + ", mem on start"); } catch (Throwable throwable) { reportIfLocal(throwable); if (throwable instanceof CascadingException) throw (CascadingException) throwable; throw new FlowException("internal error during reducer configuration", throwable); } }
From source file:cascading.flow.hadoop.stream.graph.HadoopMapStreamGraph.java
License:Open Source License
protected void buildGraph() { streamedHead = handleHead(this.source, flowProcess); Set<Tap> tributaries = ElementGraphs.findSources(elementGraph, Tap.class); tributaries.remove(this.source); // we cannot stream and accumulate the same source // accumulated paths for (Object source : tributaries) { HadoopFlowProcess hadoopProcess = (HadoopFlowProcess) flowProcess; JobConf conf = hadoopProcess.getJobConf(); // allows client side config to be used cluster side String property = conf.getRaw("cascading.node.accumulated.source.conf." + Tap.id((Tap) source)); if (property == null) throw new IllegalStateException( "accumulated source conf property missing for: " + ((Tap) source).getIdentifier()); conf = getSourceConf(hadoopProcess, conf, property); flowProcess = new HadoopFlowProcess(hadoopProcess, conf); handleHead((Tap) source, flowProcess); }/*from ww w . j av a2 s . c om*/ }
From source file:cascading.flow.hadoop.stream.HadoopMapStreamGraph.java
License:Open Source License
protected void buildGraph() { streamedHead = handleHead(this.source, flowProcess); FlowElement tail = step.getGroup() != null ? step.getGroup() : step.getSink(); Set<Tap> tributaries = step.getJoinTributariesBetween(this.source, tail); tributaries.remove(this.source); // we cannot stream and accumulate the same source // accumulated paths for (Object source : tributaries) { HadoopFlowProcess hadoopProcess = (HadoopFlowProcess) flowProcess; JobConf conf = hadoopProcess.getJobConf(); // allows client side config to be used cluster side String property = conf.getRaw("cascading.step.accumulated.source.conf." + Tap.id((Tap) source)); if (property == null) throw new IllegalStateException( "accumulated source conf property missing for: " + ((Tap) source).getIdentifier()); conf = getSourceConf(hadoopProcess, conf, property); flowProcess = new HadoopFlowProcess(hadoopProcess, conf); handleHead((Tap) source, flowProcess); }//ww w . ja va2 s . c o m }
From source file:cascading.flow.stack.FlowMapperStack.java
License:Open Source License
public FlowMapperStack(HadoopFlowProcess flowProcess) throws IOException { this.flowProcess = flowProcess; JobConf jobConf = flowProcess.getJobConf(); step = (FlowStep) Util.deserializeBase64(jobConf.getRaw("cascading.flow.step")); // is set by the MultiInputSplit currentSource = (Tap) Util.deserializeBase64(jobConf.getRaw("cascading.step.source")); if (LOG.isDebugEnabled()) LOG.debug("map current source: " + currentSource); buildStack();/* ww w .jav a2 s . c o m*/ for (Stack stack : stacks) stack.tail.open(); }
From source file:com.ibm.jaql.io.hadoop.CompositeOutputAdapter.java
License:Apache License
/** * For each entry in conf, if its value differs from the value in parent, * add it to the result JSON record./*from w w w .j a v a 2 s . com*/ * * @param conf * @param confText * @return */ private static JsonRecord saveConf(JobConf parent, JobConf conf) // TODO: move to general library { BufferedJsonRecord jrec = new BufferedJsonRecord(); // Add all entries in conf that are not in parent for (Map.Entry<String, String> entry : conf) { String name = entry.getKey(); String value = entry.getValue(); String value2 = parent.getRaw(name); if (value != value2 && !value.equals(value2)) { jrec.add(new JsonString(name), new JsonString(value)); } } // // Remove all entries in parent that are not in conf // for( Map.Entry<String, String> entry: parent ) // { // String name = entry.getKey(); // if( conf.getRaw(name) == null ) // { // jrec.add(new JsonString(name), null); // } // } return jrec; }