Example usage for org.apache.hadoop.mapred JobConf iterator

List of usage examples for org.apache.hadoop.mapred JobConf iterator

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. iterator

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf iterator.

Prototype

@Override
public Iterator<Map.Entry<String, String>> iterator()

Source Link

Document

Get an Iterator to go through the list of String key-value pairs in the configuration.

Usage

From source file:babel.prep.PrepStep.java

License:Apache License

/**
 * Dumps the configuration (for debugging only).
 *//* w w  w  . j a v  a  2  s.  co m*/
protected String dumpJobProps(JobConf job) {
    Iterator<Entry<String, String>> it = job.iterator();
    StringBuilder strBld = new StringBuilder();
    Entry<String, String> cur;

    while (it.hasNext()) {
        cur = (Entry<String, String>) it.next();
        strBld.append(cur.getKey() + " : " + cur.getValue() + "\n");
    }

    return strBld.toString();
}

From source file:com.digitalpebble.behemoth.solr.LucidWorksWriter.java

License:Apache License

public void open(JobConf job, String name) throws IOException {
    String zkHost = job.get("solr.zkhost");
    if (zkHost != null && zkHost.equals("") == false) {
        String collection = job.get("solr.zk.collection", "collection1");
        LOG.info("Indexing to collection: " + collection + " w/ ZK host: " + zkHost);
        solr = new CloudSolrServer(zkHost);
        ((CloudSolrServer) solr).setDefaultCollection(collection);
    } else {/*from  www .  ja va 2 s. c om*/
        String solrURL = job.get("solr.server.url");
        int queueSize = job.getInt("solr.client.queue.size", 100);
        int threadCount = job.getInt("solr.client.threads", 1);
        solr = new StreamingUpdateSolrServer(solrURL, queueSize, threadCount);
    }
    includeMetadata = job.getBoolean("lw.metadata", false);
    includeAnnotations = job.getBoolean("lw.annotations", false);

    // get the Behemoth annotations types and features
    // to store as SOLR fields
    // solr.f.name = BehemothType.featureName
    // e.g. solr.f.person = Person.string
    Iterator<Entry<String, String>> iterator = job.iterator();
    while (iterator.hasNext()) {
        Entry<String, String> entry = iterator.next();
        if (entry.getKey().startsWith("solr.f.") == false)
            continue;
        String fieldName = entry.getKey().substring("solr.f.".length());
        String val = entry.getValue();
        // see if a feature has been specified
        // if not we'll use '*' to indicate that we want
        // the text covered by the annotation
        HashMap<String, String> featureValMap = new HashMap<String, String>();
        int separator = val.indexOf(".");
        String featureName = "*";
        if (separator != -1)
            featureName = val.substring(separator + 1);
        featureValMap.put(featureName, fieldName);
        fieldMapping.put(entry.getValue(), featureValMap);
        LOG.debug("Adding to mapping " + entry.getValue() + " " + featureName + " " + fieldName);
    }
}

From source file:com.digitalpebble.behemoth.solr.SOLRWriter.java

License:Apache License

protected void populateSolrFieldMappingsFromBehemothAnnotationsTypesAndFeatures(JobConf job) {
    // get the Behemoth annotations types and features
    // to store as SOLR fields
    // solr.f.name = BehemothType.featureName
    // e.g. solr.f.person = Person.string will map the "string" feature of
    // "Person" annotations onto the Solr field "person"
    Iterator<Entry<String, String>> iterator = job.iterator();
    while (iterator.hasNext()) {
        Entry<String, String> entry = iterator.next();
        if (entry.getKey().startsWith("solr.f.") == false)
            continue;
        String solrFieldName = entry.getKey().substring("solr.f.".length());
        populateMapping(solrFieldName, entry.getValue());
    }/*from   w w  w  .j  a v  a  2s. c om*/

    String list = job.get("solr.annotations.list");
    if (useAnnotationPrefix) {
        if (list == null || list.trim().length() == 0)
            // Include all annotations if no annotations list is not defined
            includeAllAnnotations = true;
        else {
            // Include only annotations defined in the "solr.annotations.list" with the prefix
            String[] names = list.split("\\s+");
            for (String name : names) {
                String solrFieldName = annotationPrefix + name;
                populateMapping(solrFieldName, name);
            }
        }
    } else {
        // Include specified annotations without prefix if annotations list is defined. 
        // These fields would have to explicitly defined in Solr schema since solr.annotation.use.prefix 
        // is not defined or field mapping has to be defined
        if (list == null || list.trim().length() == 0) {
            return;
        }
        String[] names = list.split("\\s+");
        for (String name : names) {
            String solrFieldName = name;
            populateMapping(solrFieldName, name);
        }
    }

}

From source file:com.twitter.pig.backend.hadoop.executionengine.tez.TezExecutionEngine.java

License:Apache License

/**
 * Method to apply pig properties to JobConf
 * (replaces properties with resulting jobConf values)
 * @param conf JobConf with appropriate hadoop resource files
 * @param properties Pig properties that will override hadoop properties; properties might be modified
 *//*from  w w w .  j a  va  2  s.co  m*/
@SuppressWarnings("deprecation")
private void recomputeProperties(JobConf jobConf, Properties properties) {
    // We need to load the properties from the hadoop configuration
    // We want to override these with any existing properties we have.
    if (jobConf != null && properties != null) {
        // set user properties on the jobConf to ensure that defaults
        // and deprecation is applied correctly
        Enumeration<Object> propertiesIter = properties.keys();
        while (propertiesIter.hasMoreElements()) {
            String key = (String) propertiesIter.nextElement();
            String val = properties.getProperty(key);
            // We do not put user.name, See PIG-1419
            if (!key.equals("user.name"))
                jobConf.set(key, val);
        }
        //clear user defined properties and re-populate
        properties.clear();
        Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            properties.put(entry.getKey(), entry.getValue());
        }
    }
}

From source file:edu.stolaf.cs.wmrserver.streaming.PipeMapRed.java

License:Apache License

void addJobConfToEnvironment(JobConf conf, Map<String, String> env) {
    if (debug_) {
        logprintln("addJobConfToEnvironment: begin");
    }//w w w . ja  v  a2 s.c  o m
    Iterator it = conf.iterator();
    while (it.hasNext()) {
        Map.Entry en = (Map.Entry) it.next();
        String name = (String) en.getKey();
        //String value = (String)en.getValue(); // does not apply variable expansion
        String value = conf.get(name); // does variable expansion 
        name = safeEnvVarName(name);
        envPut(env, name, value);
    }
    if (debug_) {
        logprintln("addJobConfToEnvironment: end");
    }
}

From source file:hivemall.utils.hadoop.HadoopUtils.java

License:Open Source License

@Nonnull
public static String toString(@Nonnull JobConf jobconf, @Nullable String regexKey) {
    final Iterator<Entry<String, String>> itor = jobconf.iterator();
    boolean hasNext = itor.hasNext();
    if (!hasNext) {
        return "";
    }// w  w w.j  a v  a  2s.  c  o  m
    final StringBuilder buf = new StringBuilder(1024);
    do {
        Entry<String, String> e = itor.next();
        hasNext = itor.hasNext();
        String k = e.getKey();
        if (k == null) {
            continue;
        }
        if (regexKey == null || k.matches(regexKey)) {
            String v = e.getValue();
            buf.append(k).append('=').append(v);
            if (hasNext) {
                buf.append(',');
            }
        }
    } while (hasNext);
    return buf.toString();
}

From source file:org.apache.pig.backend.hadoop.executionengine.MRExecutionEngine.java

License:Apache License

/**
 * Method to apply pig properties to JobConf (replaces properties with
 * resulting jobConf values)//  w  ww  .  jav  a  2s  . com
 * 
 * @param conf
 *            JobConf with appropriate hadoop resource files
 * @param properties
 *            Pig properties that will override hadoop properties;
 *            properties might be modified
 */
@SuppressWarnings("deprecation")
protected void recomputeProperties(JobConf jobConf, Properties properties) {
    // We need to load the properties from the hadoop configuration
    // We want to override these with any existing properties we have.
    if (jobConf != null && properties != null) {
        // set user properties on the jobConf to ensure that defaults
        // and deprecation is applied correctly
        Enumeration<Object> propertiesIter = properties.keys();
        while (propertiesIter.hasMoreElements()) {
            String key = (String) propertiesIter.nextElement();
            String val = properties.getProperty(key);
            // We do not put user.name, See PIG-1419
            if (!key.equals("user.name"))
                jobConf.set(key, val);
        }
        // clear user defined properties and re-populate
        properties.clear();
        Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            properties.put(entry.getKey(), entry.getValue());
        }
    }
}

From source file:org.apache.pig.impl.util.Utils.java

License:Apache License

/**
 * Method to apply pig properties to JobConf (replaces properties with
 * resulting jobConf values).//w  ww  .j a  v a 2  s  .co m
 *
 * @param conf JobConf with appropriate hadoop resource files
 * @param properties Pig properties that will override hadoop properties;
 * properties might be modified
 */
public static void recomputeProperties(JobConf jobConf, Properties properties) {
    // We need to load the properties from the hadoop configuration
    // We want to override these with any existing properties we have.
    if (jobConf != null && properties != null) {
        // set user properties on the jobConf to ensure that defaults
        // and deprecation is applied correctly
        Enumeration<Object> propertiesIter = properties.keys();
        while (propertiesIter.hasMoreElements()) {
            String key = (String) propertiesIter.nextElement();
            String val = properties.getProperty(key);
            // We do not put user.name, See PIG-1419
            if (!key.equals("user.name")) {
                jobConf.set(key, val);
            }
        }
        // clear user defined properties and re-populate
        properties.clear();
        Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            properties.put(entry.getKey(), entry.getValue());
        }
    }
}

From source file:org.pentaho.hadoop.mapreduce.PentahoMapReduceBase.java

License:Apache License

@SuppressWarnings("unchecked")
@Override//from www . j a va2s  .c o  m
public void configure(JobConf job) {
    super.configure(job);

    debug = "true".equalsIgnoreCase(job.get("debug")); //$NON-NLS-1$

    transMapXml = job.get("transformation-map-xml");
    transCombinerXml = job.get("transformation-combiner-xml");
    transReduceXml = job.get("transformation-reduce-xml");
    mapInputStepName = job.get("transformation-map-input-stepname");
    mapOutputStepName = job.get("transformation-map-output-stepname");
    combinerInputStepName = job.get("transformation-combiner-input-stepname");
    combinerOutputStepName = job.get("transformation-combiner-output-stepname");
    combineSingleThreaded = isCombinerSingleThreaded(job);
    reduceInputStepName = job.get("transformation-reduce-input-stepname");
    reduceOutputStepName = job.get("transformation-reduce-output-stepname");
    reduceSingleThreaded = isReducerSingleThreaded(job);
    String xmlVariableSpace = job.get("variableSpace");

    if (!Const.isEmpty(xmlVariableSpace)) {
        setDebugStatus("PentahoMapReduceBase. variableSpace was retrieved from the job.  The contents: ");

        //  deserialize from xml to variable space
        XStream xStream = new XStream();

        if (xStream != null) {
            setDebugStatus("PentahoMapReduceBase: Setting classes variableSpace property.: ");
            variableSpace = (VariableSpace) xStream.fromXML(xmlVariableSpace);

            for (String variableName : variableSpace.listVariables()) {
                if (variableName.startsWith(KETTLE_VARIABLE_PREFIX)) {
                    System.setProperty(variableName, variableSpace.getVariable(variableName));
                }
            }
        }
    } else {
        setDebugStatus(
                "PentahoMapReduceBase: The PDI Job's variable space was not found in the job configuration.");
        variableSpace = new Variables();
    }

    // Check for environment variables in the userDefined variables
    Iterator<Entry<String, String>> iter = job.iterator();
    while (iter.hasNext()) {
        Entry<String, String> entry = iter.next();
        if (entry.getKey().startsWith(ENVIRONMENT_VARIABLE_PREFIX)) {
            System.setProperty(entry.getKey().substring(ENVIRONMENT_VARIABLE_PREFIX.length()),
                    entry.getValue());
        } else if (entry.getKey().startsWith(KETTLE_VARIABLE_PREFIX)) {
            System.setProperty(entry.getKey(), entry.getValue());
        }
    }

    MRUtil.passInformationToTransformation(variableSpace, job);

    switch (mrOperation) {
    case Combine:
        outClassK = (Class<K>) job.getMapOutputKeyClass();
        outClassV = (Class<V>) job.getMapOutputValueClass();
        break;
    case Reduce:
        outClassK = (Class<K>) job.getOutputKeyClass();
        outClassV = (Class<V>) job.getOutputValueClass();
        break;
    default:
        throw new IllegalArgumentException("Unsupported MapReduce operation: " + mrOperation);
    }

    if (debug) {
        System.out.println("Job configuration>");
        System.out.println("Output key class: " + outClassK.getName());
        System.out.println("Output value class: " + outClassV.getName());
    }

    //  set the log level to what the level of the job is
    String stringLogLevel = job.get("logLevel");
    if (!Const.isEmpty(stringLogLevel)) {
        logLevel = LogLevel.valueOf(stringLogLevel);
        setDebugStatus("Log level set to " + stringLogLevel);
    } else {
        System.out.println(
                "Could not retrieve the log level from the job configuration.  logLevel will not be set.");
    }

    createTrans(job);
}

From source file:org.pentaho.hadoop.mapreduce.PentahoMapRunnable.java

License:Apache License

public void configure(JobConf job) {
    pluginWaitTimeout = TimeUnit.MINUTES.toMillis(5);

    debug = "true".equalsIgnoreCase(job.get("debug")); //$NON-NLS-1$

    transMapXml = job.get("transformation-map-xml");
    transReduceXml = job.get("transformation-reduce-xml");
    mapInputStepName = job.get("transformation-map-input-stepname");
    mapOutputStepName = job.get("transformation-map-output-stepname");
    reduceInputStepName = job.get("transformation-reduce-input-stepname");
    reduceOutputStepName = job.get("transformation-reduce-output-stepname");
    String xmlVariableSpace = job.get("variableSpace");

    outClassK = (Class<K2>) job.getMapOutputKeyClass();
    outClassV = (Class<V2>) job.getMapOutputValueClass();

    if (!Const.isEmpty(xmlVariableSpace)) {
        setDebugStatus("PentahoMapRunnable(): variableSpace was retrieved from the job.  The contents: ");
        setDebugStatus(xmlVariableSpace);

        //  deserialize from xml to variable space
        XStream xStream = new XStream();

        setDebugStatus("PentahoMapRunnable(): Setting classes variableSpace property.: ");
        variableSpace = (VariableSpace) xStream.fromXML(xmlVariableSpace);

        for (String variableName : variableSpace.listVariables()) {
            if (variableName.startsWith(KETTLE_VARIABLE_PREFIX)) {
                System.setProperty(variableName, variableSpace.getVariable(variableName));
            }//w  w w  . j av a 2s.  c om
            if (KETTLE_PMR_PLUGIN_TIMEOUT.equals(variableName)) {
                try {
                    pluginWaitTimeout = Long.parseLong(variableSpace.getVariable(variableName));
                } catch (Exception e) {
                    System.out.println("Unable to parse plugin wait timeout, defaulting to 5 minutes");
                }
            }
        }
    } else {
        setDebugStatus("PentahoMapRunnable(): The PDI Job's variable space was not sent.");
        variableSpace = new Variables();
    }

    // Check for environment variables in the userDefined variables
    Iterator<Entry<String, String>> iter = job.iterator();
    while (iter.hasNext()) {
        Entry<String, String> entry = iter.next();
        if (entry.getKey().startsWith(ENVIRONMENT_VARIABLE_PREFIX)) {
            System.setProperty(entry.getKey().substring(ENVIRONMENT_VARIABLE_PREFIX.length()),
                    entry.getValue());
        } else if (entry.getKey().startsWith(KETTLE_VARIABLE_PREFIX)) {
            System.setProperty(entry.getKey(), entry.getValue());
        }
    }

    MRUtil.passInformationToTransformation(variableSpace, job);

    setDebugStatus("Job configuration");
    setDebugStatus("Output key class: " + outClassK.getName());
    setDebugStatus("Output value class: " + outClassV.getName());

    //  set the log level to what the level of the job is
    String stringLogLevel = job.get("logLevel");
    if (!Const.isEmpty(stringLogLevel)) {
        logLevel = LogLevel.valueOf(stringLogLevel);
        setDebugStatus("Log level set to " + stringLogLevel);
    } else {
        System.out.println(
                "Could not retrieve the log level from the job configuration.  logLevel will not be set.");
    }

    long deadline = 0;
    boolean first = true;
    while (true) {
        createTrans(job);

        if (first) {
            deadline = pluginWaitTimeout + System.currentTimeMillis();
            System.out.println(PentahoMapRunnable.class + ": Trans creation checking starting now "
                    + new Date().toString());
            first = false;
        }

        List<MissingTrans> missingTranses = new ArrayList<MissingTrans>();
        for (StepMeta stepMeta : trans.getTransMeta().getSteps()) {
            StepMetaInterface stepMetaInterface = stepMeta.getStepMetaInterface();
            if (stepMetaInterface instanceof MissingTrans) {
                MissingTrans missingTrans = (MissingTrans) stepMetaInterface;
                System.out.println(MissingTrans.class + "{stepName: " + missingTrans.getStepName()
                        + ", missingPluginId: " + missingTrans.getMissingPluginId() + "}");
                missingTranses.add(missingTrans);
            }
        }

        if (missingTranses.size() == 0) {
            System.out.println(
                    PentahoMapRunnable.class + ": Done waiting on plugins now " + new Date().toString());
            break;
        } else {
            if (System.currentTimeMillis() > deadline) {
                StringBuilder stringBuilder = new StringBuilder("Failed to initialize plugins: ");
                for (MissingTrans missingTrans : missingTranses) {
                    stringBuilder.append(missingTrans.getMissingPluginId());
                    stringBuilder.append(" on step ").append(missingTrans.getStepName());
                    stringBuilder.append(", ");
                }
                stringBuilder.setLength(stringBuilder.length() - 2);
                throw new RuntimeException(stringBuilder.toString());
            } else {
                try {
                    Thread.sleep(Math.min(100, deadline - System.currentTimeMillis()));
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
        }
    }
}