List of usage examples for org.apache.hadoop.mapred JobConf get
public String get(String name)
name property, null if no such property exists. From source file:net.peacesoft.nutch.crawl.ReSolrWriter.java
License:Apache License
void init(SolrServer server, JobConf job) throws IOException { solr = server;//from w ww.j a va 2s. co m commitSize = job.getInt(SolrConstants.COMMIT_SIZE, 1000); solrMapping = SolrMappingReader.getInstance(job); delete = job.getBoolean(IndexerMapReduce.INDEXER_DELETE, false); // parse optional params params = new ModifiableSolrParams(); String paramString = job.get(SolrConstants.PARAMS); if (paramString != null) { String[] values = paramString.split("&"); for (String v : values) { String[] kv = v.split("="); if (kv.length < 2) { continue; } params.add(kv[0], kv[1]); } } }
From source file:nl.tudelft.graphalytics.mapreducev2.bfs.DirectedBreadthFirstSearchMap.java
License:Apache License
public void configure(JobConf job) { srcId = job.get(BreadthFirstSearchConfiguration.SOURCE_VERTEX_KEY); }
From source file:nl.tudelft.graphalytics.mapreducev2.bfs.UndirectedBreadthFirstSearchMap.java
License:Apache License
public void configure(JobConf job) { srcId = Long.parseLong(job.get(BreadthFirstSearchConfiguration.SOURCE_VERTEX_KEY)); }
From source file:nl.tudelft.graphalytics.mapreducev2.cd.DirectedCambridgeLPAReducer.java
License:Apache License
public void configure(JobConf job) { this.deltaParam = Float.parseFloat(job.get(CommunityDetectionConfiguration.HOP_ATTENUATION)); this.mParam = Float.parseFloat(job.get(CommunityDetectionConfiguration.NODE_PREFERENCE)); }
From source file:nl.tudelft.graphalytics.mapreducev2.evo.DirectedForestFireModelMap.java
License:Apache License
@Override public void configure(JobConf conf) { TaskAttemptID attempt = TaskAttemptID.forName(conf.get("mapred.task.id")); this.taskID = attempt.getTaskID().getId(); // todo verify this.newVerticesPerSlot = conf.getInt(ForestFireModelUtils.NEW_VERTICES_NR, -1); this.maxID = conf.getLong(ForestFireModelUtils.MAX_ID, -1); this.isFirst = conf.getBoolean(ForestFireModelUtils.IS_INIT, false); this.isInit = this.isFirst; if (this.isInit) this.ambassadors = new HashMap<LongWritable, List<LongWritable>>(); else// w ww. ja v a 2 s. c o m this.ambassadors = ForestFireModelUtils .verticesIdsString2Map(conf.get(ForestFireModelUtils.CURRENT_AMBASSADORS)); }
From source file:nl.tudelft.graphalytics.mapreducev2.evo.UndirectedForestFireModelMap.java
License:Apache License
@Override public void configure(JobConf conf) { TaskAttemptID attempt = TaskAttemptID.forName(conf.get("mapred.task.id")); this.taskID = attempt.getTaskID().getId(); this.newVerticesPerSlot = conf.getInt(ForestFireModelUtils.NEW_VERTICES_NR, -1); this.maxID = conf.getLong(ForestFireModelUtils.MAX_ID, -1); this.isFirst = conf.getBoolean(ForestFireModelUtils.IS_INIT, false); this.isInit = this.isFirst; if (this.isInit) this.ambassadors = new HashMap<LongWritable, List<LongWritable>>(); else//from w ww . j av a 2 s . co m this.ambassadors = ForestFireModelUtils .verticesIdsString2Map(conf.get(ForestFireModelUtils.CURRENT_AMBASSADORS)); }
From source file:oracle.kv.hadoop.hive.table.TableHiveRecordReader.java
License:Open Source License
private void initialize(final JobConf jobConf, final TableRecordReader v2RecRdr) { LOG.trace("open"); /*//from w w w . jav a2 s .c o m * Initialize any information related to predicate pushdown. * * Note that this class is instantiated, and its methods are invoked, * on the server side of a given Hive query; from a process running * on a MapReduce job's DataNode in a Hadoop cluster. When that * query includes a predicate (a WHERE clause) the predicate is * parsed and analyzed on the client side. If the predicate consists * of components that can be pushed to the KVStore for backend * filtering, then the client side decomposes the predicate into * two disjoint sets; one set containing the search conditions that * will be pushed and the other containing the remaining ("residual") * parts of the original predicate. When the query is successfully * decomposed, the search conditions are serialized and sent to the * server side (the MapReduce job) for further processing here; * whereas the residual predicate remains on the client side and * will be applied to the results produced by the filtering performed * by the MapReduce job on the backend. * * To communicate the search conditions (the predicate to be pushed) * from the client side to the server side, the client side Hive * infrastructure sets a property with name FILTER_EXPR_CONF_STR * to a value consisting of the SERIALIZED form of the search * conditions. The value of that property is retrieved below in * this method, and then deserialized to recover the search * conditions to push to the KVStore server side so that filtering * can be performed there. * * Note that because the residual predicate will not be pushed, Hive * only sends the search conditions; never the residual. Thus, the * residual predicate will always be null (and ignored) in this * class. */ final String filterExprPropVal = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR); if (filterExprPropVal != null) { final ExprNodeGenericFuncDesc filterExpr = Utilities.deserializeExpression(filterExprPropVal); if (LOG.isTraceEnabled()) { LOG.trace("Hive query = " + HiveConf.getVar(jobConf, HiveConf.ConfVars.HIVEQUERYSTRING)); LOG.trace("deserialized predicate = " + filterExpr); final StringBuilder whereClause = new StringBuilder(); TableHiveInputFormat.buildPushPredicate(filterExpr, whereClause); LOG.trace("where clause = " + whereClause.toString()); } final List<String> hiveFilterCols = filterExpr.getCols(); if (hiveFilterCols != null) { final IndexPredicateAnalyzer analyzer = TableHiveInputFormat.basicPredicateAnalyzer(hiveFilterCols); final List<IndexSearchCondition> hiveSearchConditions = new ArrayList<IndexSearchCondition>(); /* * The analyzePredicate method will use the predicate info * retrieved from the deserialized filterExpr to populate the * hiveSearchConditions ArrayList with the desired search * conditions. That method also returns the residual predicate. * But in this case, since the residual will always be null * (as explained above), the return value is ignored below. */ analyzer.analyzePredicate(filterExpr, hiveSearchConditions); if (LOG.isTraceEnabled()) { LOG.trace("search conditions to validate = " + hiveSearchConditions); } final Table table = v2RecRdr.getKvTable(); if (table != null) { /* * If the search conditions retrieved from the deserialized * property can be pushed (are valid), then construct the * appropriate key (IndexKey or PrimaryKey) and/or * FieldRange. */ if (TableHiveInputFormat.searchConditionsValid(hiveSearchConditions, table)) { /* Give index priority over primaryKey. */ FieldRange fieldRange = null; PrimaryKey primaryKey = null; final IndexKey indexKey = TableHiveInputFormat .indexKeyFromSearchConditionsNoRange(hiveSearchConditions, table); if (indexKey != null) { v2RecRdr.setIndexKey(indexKey); fieldRange = indexFieldRangeFromSearchConditions(hiveSearchConditions, indexKey.getIndex()); } else { primaryKey = primaryKeyFromSearchConditionsNoRange(hiveSearchConditions, table); if (primaryKey != null) { v2RecRdr.setPrimaryKey(primaryKey); fieldRange = primaryFieldRangeFromSearchConditions(hiveSearchConditions, table); } } v2RecRdr.setMultiRowOptions(fieldRange); if (LOG.isTraceEnabled()) { LOG.trace("primaryKey = " + primaryKey); LOG.trace("indexKey = " + indexKey); } } else { LOG.trace("search conditions not valid for " + "PrimaryKey or IndexKey iteration. " + "Convert predicate to native query string"); } } } } this.v2RecordReader = v2RecRdr; }
From source file:oracle.kv.hadoop.hive.table.V1V2TableUtil.java
License:Open Source License
/** * Convenience method that sets the properties the current Hive query * job needs to connect to and retrieve records from the store; * specifically, the name of the store and the store's helper hosts. * <p>/*from w ww.ja v a2 s . c o m*/ * Implementation Note: if the Hive query to be performed is a * "client-side query" -- that is, a query in which the processing * occurs only on the Hive client, not via a MapReduce job -- then * the values handled by this method should be already set; via the * TBLPROPERTIES entered on the Hive command line. On the other * hand, if the query is complex enough that the Hive infrastructure * must construct and submit a MapReduce job to perform the query, * then the values set by this method are obtained from the given * InputSplit; which must be Non-Null. */ private static void getStoreInfo(final JobConf jobConf, final TableHiveInputSplit split) { /* 1. Store name */ String storeName = jobConf.get(ParamConstant.KVSTORE_NAME.getName()); if (storeName == null) { /* Must be MapReduce: get store name from split if it exists. */ if (split != null) { storeName = split.getKVStoreName(); if (storeName != null) { jobConf.set(ParamConstant.KVSTORE_NAME.getName(), storeName); } } } /* 2. Helper Hosts */ String hostsStr = jobConf.get(ParamConstant.KVSTORE_NODES.getName()); if (hostsStr == null) { /* Must be MapReduce: get store nodes from split if it exists. */ if (split != null) { final String[] hostsArray = split.getKVHelperHosts(); if (hostsArray != null) { final StringBuilder buf = new StringBuilder(hostsArray[0]); for (int i = 1; i < hostsArray.length; i++) { buf.append("," + hostsArray[i]); } hostsStr = buf.toString(); jobConf.set(ParamConstant.KVSTORE_NODES.getName(), hostsStr); } } } /* 3. Table name */ String tableName = jobConf.get(ParamConstant.TABLE_NAME.getName()); if (tableName == null) { /* Must be MapReduce: get table name from split if it exists. */ if (split != null) { tableName = split.getTableName(); if (tableName != null) { jobConf.set(ParamConstant.TABLE_NAME.getName(), tableName); } } } /* * 4. Security artifacts, loginFile, trustFile, PasswordCredentials. * Note that if any of the properties retrieved below that * correspond to a file is either not set (null), or is set to * a path that cannot be found on the local file system, then * assume it must be a MapReduce job, and try to get the name * of the file and credentials from the split (if it exists). * If those artifacts cannot be obtained, then try to execute * without security. */ /* 4a. Login file name. */ String loginFile = jobConf.get(KVSecurityConstants.SECURITY_FILE_PROPERTY); if (loginFile == null) { if (split != null) { loginFile = split.getSecurityLogin(); if (loginFile != null) { jobConf.set(KVSecurityConstants.SECURITY_FILE_PROPERTY, loginFile); } } } else { final File loginFileFd = new File(loginFile); if (!loginFileFd.exists()) { if (split != null) { loginFile = split.getSecurityLogin(); jobConf.set(KVSecurityConstants.SECURITY_FILE_PROPERTY, loginFile); } } else { jobConf.set(KVSecurityConstants.SECURITY_FILE_PROPERTY, loginFile); } } /* 4b.Trust file name. */ String trustFile = jobConf.get(KVSecurityConstants.SSL_TRUSTSTORE_FILE_PROPERTY); if (trustFile == null) { if (split != null) { trustFile = split.getSecurityTrust(); if (trustFile != null) { jobConf.set(KVSecurityConstants.SSL_TRUSTSTORE_FILE_PROPERTY, trustFile); } } } else { final File trustFileFd = new File(trustFile); if (!trustFileFd.exists()) { if (split != null) { trustFile = split.getSecurityTrust(); jobConf.set(KVSecurityConstants.SSL_TRUSTSTORE_FILE_PROPERTY, trustFile); } } else { jobConf.set(KVSecurityConstants.SSL_TRUSTSTORE_FILE_PROPERTY, trustFile); } } /* 4c. PasswordCredentials (username and password). */ final String walletLoc = jobConf.get(KVSecurityConstants.AUTH_WALLET_PROPERTY); final String pwdLoc = jobConf.get(KVSecurityConstants.AUTH_PWDFILE_PROPERTY); if (walletLoc == null && pwdLoc == null) { if (split != null) { final PasswordCredentials passwordCredentials = split.getSecurityCredentials(); if (passwordCredentials != null) { jobConf.set(KVSecurityConstants.AUTH_USERNAME_PROPERTY, passwordCredentials.getUsername()); jobConf.set(ParamConstant.AUTH_USER_PWD_PROPERTY.getName(), String.valueOf(passwordCredentials.getPassword())); } } } if (walletLoc != null) { final File walletLocFd = new File(walletLoc); if (!walletLocFd.exists()) { if (split != null) { final PasswordCredentials passwordCredentials = split.getSecurityCredentials(); if (passwordCredentials != null) { jobConf.set(KVSecurityConstants.AUTH_USERNAME_PROPERTY, passwordCredentials.getUsername()); jobConf.set(ParamConstant.AUTH_USER_PWD_PROPERTY.getName(), String.valueOf(passwordCredentials.getPassword())); } } } } else if (pwdLoc != null) { final File pwdLocFd = new File(pwdLoc); if (!pwdLocFd.exists()) { if (split != null) { final PasswordCredentials passwordCredentials = split.getSecurityCredentials(); if (passwordCredentials != null) { jobConf.set(KVSecurityConstants.AUTH_USERNAME_PROPERTY, passwordCredentials.getUsername()); jobConf.set(ParamConstant.AUTH_USER_PWD_PROPERTY.getName(), String.valueOf(passwordCredentials.getPassword())); } } } } }
From source file:org.acacia.csr.java.VertexPartitioner.java
License:Apache License
public void configure(JobConf arg0) { // TODO Auto-generated method stub numberOfVerts = Long.parseLong(arg0.get("vertex-count")); }
From source file:org.apache.ambari.servicemonitor.jobs.FileUsingJobRunner.java
License:Apache License
public int run(String[] args) throws Exception { // Configuration processed by ToolRunner Configuration conf = getConf(); CommandLine commandLine = getCommandLine(); // Create a JobConf using the processed conf JobConf jobConf = new JobConf(conf, FileUsingJobRunner.class); //tune the config if (jobConf.get(JobKeys.RANGEINPUTFORMAT_ROWS) == null) { jobConf.setInt(JobKeys.RANGEINPUTFORMAT_ROWS, 1); }// w ww. j ava 2 s . co m // Process custom command-line options String name = OptionHelper.getStringOption(commandLine, "n", "File Using Job"); if (commandLine.hasOption('x')) { //delete the output directory String destDir = jobConf.get(JobKeys.MAPRED_OUTPUT_DIR); FileSystem fs = FileSystem.get(jobConf); fs.delete(new Path(destDir), true); } // Specify various job-specific parameters jobConf.setMapperClass(FileUsingMapper.class); jobConf.setReducerClass(FileUsingReducer.class); jobConf.setMapOutputKeyClass(IntWritable.class); jobConf.setMapOutputValueClass(IntWritable.class); jobConf.setOutputFormat(TextOutputFormat.class); jobConf.setInputFormat(RangeInputFormat.class); //jobConf.setPartitionerClass(SleepJob.class); jobConf.setSpeculativeExecution(false); jobConf.setJobName(name); jobConf.setJarByClass(this.getClass()); FileInputFormat.addInputPath(jobConf, new Path("ignored")); // Submit the job, then poll for progress until the job is complete RunningJob runningJob = JobClient.runJob(jobConf); runningJob.waitForCompletion(); return runningJob.isSuccessful() ? 0 : 1; }