Example usage for org.apache.hadoop.mapred JobConf get

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:edu.yale.cs.hadoopdb.connector.DBInputFormat.java

License:Apache License

/**
 * Retrieves the location of chunks for a given
 * relation. Then, it creates as many splits as the number of chunks. Each split is assigned
 * a chunk (which holds connection and location information).
 *//*from   w  ww .  j a v  a  2 s.  c  o m*/
@Override
public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException {

    Catalog.getInstance(conf).setSplitLocationStructure(dbConf, conf.get(DBConst.DB_RELATION_ID));
    Collection<DBChunk> chunks = dbConf.getChunks();
    InputSplit[] splits = new InputSplit[chunks.size()];

    int i = 0;
    for (DBChunk chunk : chunks) {
        DBInputSplit split = new DBInputSplit();
        split.setChunk(chunk);
        split.setRelation(dbConf.getRelation());

        splits[i] = split;
        i++;
    }

    return splits;
}

---

From source file:edu.yale.cs.hadoopdb.exec.DBJobBase.java

License:Apache License

/**
 * Job config initialization (command-line params etc).  
 *//*from  w  ww  .  j  ava  2 s. co  m*/
protected JobConf initConf(String[] args) throws Exception {

    List<String> other_args = new ArrayList<String>();

    Path configuration_file = null;
    boolean replication = false;

    for (int i = 0; i < args.length; ++i) {
        if (("-" + DBConst.DB_CONFIG_FILE).equals(args[i])) {
            configuration_file = new Path(args[++i]);
        } else if ("-replication".equals(args[i])) {
            replication = true;
        } else {
            other_args.add(args[i]);
        }
    }

    JobConf conf = null;

    conf = configureJob(other_args.toArray(new String[0]));
    LOG.info(conf.getJobName());
    LOG.info(conf.get(DBConst.DB_SQL_QUERY));

    if (conf.get(DBConst.DB_RELATION_ID) == null || conf.get(DBConst.DB_SQL_QUERY) == null
            || conf.get(DBConst.DB_RECORD_READER) == null) {
        throw new Exception(
                "ERROR: DB Job requires a relation, an SQL Query and a Record Reader class to be configured.\n"
                        + "Please specify using: conf.set(\"" + DBConst.DB_RELATION_ID
                        + "\", <relation name>), conf.set(\"" + DBConst.DB_SQL_QUERY + "\", <SQL QUERY>)\n"
                        + "and code an appropriate Record Reader and specify conf.set(\""
                        + DBConst.DB_RECORD_READER + "\", <Record reader class name>)\n");
    }

    if (replication) {
        conf.setBoolean(DBConst.DB_REPLICATION, true);
    }

    if (configuration_file == null) {
        if (conf.get(DBConst.DB_CONFIG_FILE) == null) {
            throw new Exception("No HadoopDB config file!");
        }
    } else {
        conf.set(DBConst.DB_CONFIG_FILE, configuration_file.toString());
    }

    setInputFormat(conf);

    return conf;
}

---

From source file:edu.yale.cs.hadoopdb.sms.connector.SMSInputFormat.java

License:Apache License

/**
 * Retrieves path information from FileInputFormat super class and then
 * relation from path using the job configuration. Obtains the chunk locations
 * from the HadoopDB catalog for the given relation and then creates a split for each chunk
 * of a relation. The splits are provided with path, chunk and relation.
 *///from  www .  ja  va  2s  . c om
@Override
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    // This is why we need FileInputFormat!!
    Path p = getInputPaths(job)[0];
    String p_alias = p.toUri().getPath();

    LOG.debug("Processing splits for: " + p_alias);
    String relation = job.get(p_alias).toLowerCase();

    if (!rel_DBConf.containsKey(relation)) {
        LOG.debug("Generating split structure for relation: " + relation);
        SMSConfiguration smsConf = new SMSConfiguration();
        Catalog.getInstance(job).setSplitLocationStructure(smsConf, relation);
        rel_DBConf.put(relation, smsConf);
    }

    SMSConfiguration DBConf = rel_DBConf.get(relation);
    Collection<DBChunk> chunks = DBConf.getChunks();

    InputSplit[] splits = new InputSplit[chunks.size()];
    int i = 0;

    for (DBChunk chunk : chunks) {
        SMSInputSplit split = new SMSInputSplit();
        split.setPath(p);
        split.setRelation(relation);
        split.setChunk(chunk);
        splits[i] = split;
        i++;
    }
    return splits;
}

---

From source file:edu.yale.cs.hadoopdb.sms.connector.SMSRecordReader.java

License:Apache License

/**
 * Each relation is associated with a SQL query and schema in 
 * the job configuration. This is retrieved and a {@link ParseSchema} object
 * is created to retrieve the required fields from the result set
 * and serialize them into a delimited string.
 *///from  w w  w . j  a  v a  2 s  .c  o m
public SMSRecordReader(SMSInputSplit split, JobConf conf) throws SQLException {

    this.split = split;
    this.conf = conf;
    parser = new ParseSchema(conf.get(SMSInputFormat.DB_QUERY_SCHEMA_PREFIX + "_" + split.getRelation()));

    setupDB(split, conf);
}

---

From source file:eu.larkc.iris.storage.FactsConfigurationFactory.java

License:Apache License

@SuppressWarnings("unchecked")
public static IFactsConfiguration createFactsConfiguration(JobConf jobConf) {
    String factsConfigurationClass = jobConf.get(IFactsConfiguration.FACTS_CONFIGURATION_CLASS);
    Class<? extends IFactsConfiguration> clazz = null;
    try {//  ww w .  j a  va2  s  .c o m
        clazz = (Class<? extends IFactsConfiguration>) Class.forName(factsConfigurationClass);
    } catch (ClassNotFoundException e) {
        logger.error("class not found " + factsConfigurationClass, e);
        throw new RuntimeException("class not found " + factsConfigurationClass, e);
    }
    try {
        IFactsConfiguration factsConfiguration = clazz.newInstance();
        Properties storageProperties = new Properties();
        storageProperties.load(FactsConfigurationFactory.class.getResourceAsStream(STORAGE_PROPERTIES));
        factsConfiguration.setJobConf(jobConf);
        factsConfiguration.setStorageProperties(storageProperties);
        factsConfigurations.put(jobConf, factsConfiguration);
        return factsConfiguration;
    } catch (Exception e) {
        logger.error("exception creating instance from class", e);
        throw new RuntimeException("exception creating instance from class", e);
    }
}

---

From source file:eu.larkc.iris.storage.rdf.RdfFactsRecordReader.java

License:Apache License

public RdfFactsRecordReader(InputSplit split, Class<T> inputClass, JobConf job) {
    super(split, inputClass, job);

    FactsInputSplit factsInputSplit = (FactsInputSplit) split;
    factsStorage = new RdfStorage();
    RdfFactsConfiguration rdfFactsConfiguration = (RdfFactsConfiguration) FactsConfigurationFactory
            .getFactsConfiguration(job);
    ModelSet modelSet = rdfFactsConfiguration.getModelSet(true);
    ((RdfStorage) factsStorage).setModel(modelSet);
    ((RdfStorage) factsStorage).setPredicateFilter(job.get(IFactsConfiguration.PREDICATE_FILTER));
    ((RdfStorage) factsStorage).setLimit(factsInputSplit.getLimit());
    ((RdfStorage) factsStorage).setOffset(factsInputSplit.getOffset());
}

---

From source file:eu.scape_project.tb.chutney.Chutney.java

License:Apache License

/**
 * Method run at the beginning of the map.  Note this is where we can recover settings
 * passed to us by the parent class like job type and job name.
 *//*from ww  w . j  a v  a2  s .co  m*/
@Override
public void configure(JobConf pJob) {
    super.configure(pJob);

    //get the job name from the config
    gJobName = pJob.getJobName();
    //outputPath = job.get(Settings.OUTPUTPATH_CONF_SETTING);

    //get the type of job we are running from the config
    String jobType = pJob.get(Settings.JOBTYPE_CONF_SETTING);
    //could iterate over the enum instead?
    if (jobType.equals(JobType.CommandLineJob.toString())) {
        gJobType = JobType.CommandLineJob;
    } else if (jobType.equals(JobType.TavernaCommandLine.toString())) {
        gJobType = JobType.TavernaCommandLine;
    } else if (jobType.equals(JobType.TavernaServerJob.toString())) {
        gJobType = JobType.TavernaServerJob;
    } else if (jobType.equals(JobType.XMLWorkflowReport.toString())) {
        gJobType = JobType.XMLWorkflowReport;
    } else if (jobType.equals(JobType.XMLCommandLineJob.toString())) {
        gJobType = JobType.XMLCommandLineJob;
        gXmlCode = pJob.get(Settings.XMLCODE_CONF_SETTING);
    }

}

---

From source file:fm.last.hbase.mapred.TypedBytesTableInputFormat.java

License:Apache License

public void configure(JobConf job) {
    Path[] tableNames = FileInputFormat.getInputPaths(job);
    String colArg = job.get(COLUMN_LIST);
    String[] colNames = colArg.split(" ");
    byte[][] m_cols = new byte[colNames.length][];
    for (int i = 0; i < m_cols.length; i++) {
        m_cols[i] = Bytes.toBytes(colNames[i]);
    }//from   w w  w .j  av  a  2  s  . c o m
    setInputColumns(m_cols);
    try {
        setHTable(new HTable(new HBaseConfiguration(job), tableNames[0].getName()));
    } catch (Exception e) {
        LOG.error(StringUtils.stringifyException(e));
    }
}

---

From source file:fm.last.hbase.mapred.TypedBytesTableInputFormat.java

License:Apache License

public void validateInput(JobConf job) throws IOException {
    // expecting exactly one path
    Path[] tableNames = FileInputFormat.getInputPaths(job);
    if (tableNames == null || tableNames.length > 1) {
        throw new IOException("expecting one table name");
    }/*from w  w  w  .  j a v a2s.  c om*/

    // connected to table?
    if (getHTable() == null) {
        throw new IOException("could not connect to table '" + tableNames[0].getName() + "'");
    }

    // expecting at least one column
    String colArg = job.get(COLUMN_LIST);
    if (colArg == null || colArg.length() == 0) {
        throw new IOException("expecting at least one column");
    }

}

---

From source file:fm.last.hbase.mapred.TypedBytesTableOutputFormat.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public RecordWriter getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress)
        throws IOException {

    // expecting exactly one path

    String tableName = job.get(OUTPUT_TABLE);
    HTable table = null;/*from   w  w  w.ja  v a 2s.c  o m*/
    try {
        table = new HTable(new HBaseConfiguration(job), tableName);
    } catch (IOException e) {
        LOG.error(e);
        throw e;
    }
    table.setAutoFlush(false);
    return new TableRecordWriter(table);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• »