List of usage examples for org.apache.hadoop.mapred JobConf getBoolean
public boolean getBoolean(String name, boolean defaultValue)
name property as a boolean. From source file:StreamWikiDumpInputFormat.java
License:Apache License
public RecordReader<Text, Text> getRecordReader(final InputSplit genericSplit, JobConf job, Reporter reporter) throws IOException { // handling non-standard record reader (likely StreamXmlRecordReader) FileSplit split = (FileSplit) genericSplit; LOG.info("getRecordReader start.....split=" + split); reporter.setStatus(split.toString()); // Open the file and seek to the start of the split FileSystem fs = split.getPath().getFileSystem(job); String patt = job.get(KEY_EXCLUDE_PAGE_PATTERN); boolean prev = job.getBoolean(KEY_PREVIOUS_REVISION, true); return new MyRecordReader(split, reporter, job, fs, patt != null && !"".equals(patt) ? Pattern.compile(patt) : null, prev); }
From source file:SleepJobWithArray.java
License:Apache License
public void configure(JobConf job) { this.mapSleepCount = job.getInt("sleep.job.map.sleep.count", mapSleepCount); this.initBigArray = job.getBoolean("initBigArray", false); this.bigArraySize = job.getInt("bigArraySize", bigArraySize); this.reduceSleepCount = job.getInt("sleep.job.reduce.sleep.count", reduceSleepCount); this.mapSleepDuration = job.getLong("sleep.job.map.sleep.time", 100) / mapSleepCount; this.reduceSleepDuration = job.getLong("sleep.job.reduce.sleep.time", 100) / reduceSleepCount; }
From source file:cascading.flow.stack.StackElement.java
License:Open Source License
private static TapCollector getTrapCollector(Tap trap, JobConf jobConf) { TapCollector trapCollector = trapCollectors.get(trap); if (trapCollector == null) { try {// ww w . j a v a2s. c o m jobConf = new JobConf(jobConf); int id = jobConf.getInt("cascading.flow.step.id", 0); String partname; if (jobConf.getBoolean("mapred.task.is.map", true)) partname = String.format("-m-%05d-", id); else partname = String.format("-r-%05d-", id); jobConf.set("cascading.tapcollector.partname", "%s%spart" + partname + "%05d"); trapCollector = (TapCollector) trap.openForWrite(jobConf); trapCollectors.put(trap, trapCollector); } catch (IOException exception) { throw new StackException(exception); } } return trapCollector; }
From source file:cascading.hbase.helper.TableInputFormat.java
License:Apache License
public void configure(JobConf job) { // Path[] tableNames = FileInputFormat.getInputPaths(job); // String colArg = job.get(COLUMN_LIST); // String[] colNames = colArg.split(" "); // byte [][] m_cols = new byte[colNames.length][]; // for (int i = 0; i < m_cols.length; i++) { // m_cols[i] = Bytes.toBytes(colNames[i]); // }//from w w w . ja va 2s .c o m // setInputColumns(m_cols); // try { // setHTable(new HTable(HBaseConfiguration.create(job), tableNames[0].getName())); // } catch (Exception e) { // LOG.error(StringUtils.stringifyException(e)); // } //this.conf = configuration; String tableName = job.get(INPUT_TABLE); try { setHTable(new HTable(new Configuration(job), tableName)); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } Scan scan = null; if (job.get(SCAN) != null) { try { scan = convertStringToScan(job.get(SCAN)); } catch (IOException e) { LOG.error("An error occurred.", e); } } else { try { scan = new Scan(); if (job.get(SCAN_COLUMNS) != null) { addColumns(scan, job.get(SCAN_COLUMNS)); } if (job.get(SCAN_COLUMN_FAMILY) != null) { scan.addFamily(Bytes.toBytes(job.get(SCAN_COLUMN_FAMILY))); } if (job.get(SCAN_TIMESTAMP) != null) { scan.setTimeStamp(Long.parseLong(job.get(SCAN_TIMESTAMP))); } if (job.get(SCAN_TIMERANGE_START) != null && job.get(SCAN_TIMERANGE_END) != null) { scan.setTimeRange(Long.parseLong(job.get(SCAN_TIMERANGE_START)), Long.parseLong(job.get(SCAN_TIMERANGE_END))); } if (job.get(SCAN_MAXVERSIONS) != null) { scan.setMaxVersions(Integer.parseInt(job.get(SCAN_MAXVERSIONS))); } if (job.get(SCAN_CACHEDROWS) != null) { scan.setCaching(Integer.parseInt(job.get(SCAN_CACHEDROWS))); } // false by default, full table scans generate too much BC churn scan.setCacheBlocks((job.getBoolean(SCAN_CACHEBLOCKS, false))); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } } setScan(scan); }
From source file:cascading.tap.hadoop.Hadoop18TapUtil.java
License:Open Source License
/** * should only be called if not in a Flow * * @param conf/*from ww w . j av a2 s .c om*/ * @throws IOException */ public static void setupJob(JobConf conf) throws IOException { Path outputPath = FileOutputFormat.getOutputPath(conf); if (outputPath == null) return; if (getFSSafe(conf, outputPath) == null) return; if (conf.get("mapred.task.id") == null) // need to stuff a fake id { String mapper = conf.getBoolean("mapred.task.is.map", true) ? "m" : "r"; conf.set("mapred.task.id", String.format("attempt_%012d_0000_%s_000000_0", (int) Math.rint(System.currentTimeMillis()), mapper)); } makeTempPath(conf); if (writeDirectlyToWorkingPath(conf, outputPath)) { LOG.info("writing directly to output path: " + outputPath); setWorkOutputPath(conf, outputPath); return; } // "mapred.work.output.dir" Path taskOutputPath = getTaskOutputPath(conf); setWorkOutputPath(conf, taskOutputPath); }
From source file:cascading.tap.hadoop.Hadoop18TapUtil.java
License:Open Source License
/** used in AWS EMR to disable temp paths on some file systems, s3. */ private static boolean writeDirectlyToWorkingPath(JobConf conf, Path path) { FileSystem fs = getFSSafe(conf, path); if (fs == null) return false; boolean result = conf.getBoolean("mapred.output.direct." + fs.getClass().getSimpleName(), false); if (result)/*from w ww . j av a2 s.c o m*/ LOG.info("output direct is enabled for this fs: " + fs.getName()); return result; }
From source file:co.nubetech.hiho.mapred.MySQLLoadDataMapper.java
License:Apache License
@Override public void configure(JobConf job) { try {/*from ww w .j av a2 s . c o m*/ Class.forName("com.mysql.jdbc.Driver").newInstance(); String connString = job.get(DBConfiguration.URL_PROPERTY); String username = job.get(DBConfiguration.USERNAME_PROPERTY); String password = job.get(DBConfiguration.PASSWORD_PROPERTY); logger.debug("Connection values are " + connString + " " + username + "/" + password); connect(connString, username, password); } catch (Exception e) { e.printStackTrace(); } querySuffix = job.get(HIHOConf.LOAD_QUERY_SUFFIX); hasHeaderLine = job.getBoolean(HIHOConf.LOAD_HAS_HEADER, false); keyIsTableName = job.getBoolean(HIHOConf.LOAD_KEY_IS_TABLENAME, false); disableKeys = job.getBoolean(HIHOConf.LOAD_DISABLE_KEYS, false); }
From source file:com.alexholmes.hadooputils.combine.common.mapred.SplitMetricsCombineInputFormat.java
License:Apache License
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { InputSplit[] splits = super.getSplits(job, numSplits); if (job.getBoolean("hadooputils.combine.sink.enabled", false)) { writeSplitsToSink(job, organizeSplitsByLocation(splits)); }//from w w w. j a v a 2s.c o m return splits; }
From source file:com.benchmark.mapred.terasort.TeraOutputFormat.java
License:Apache License
/** * Does the user want a final sync at close? *///w w w . j ava2s. co m public static boolean getFinalSync(JobConf conf) { return conf.getBoolean(FINAL_SYNC_ATTRIBUTE, false); }
From source file:com.chriscx.mapred.Map.java
public void configure(JobConf job) { caseSensitive = job.getBoolean("wordcount.case.sensitive", true); inputFile = job.get("map.input.file"); if (job.getBoolean("wordcount.skip.patterns", false)) { Path[] patternsFiles = new Path[0]; try {//from w w w . ja v a 2 s. c o m patternsFiles = DistributedCache.getLocalCacheFiles(job); } catch (IOException ioe) { System.err.println( "Caught exception while getting cached files: " + StringUtils.stringifyException(ioe)); } for (Path patternsFile : patternsFiles) { parseSkipFile(patternsFile); } } }