List of usage examples for org.apache.hadoop.mapred JobConf getInt
public int getInt(String name, int defaultValue)
name property as an int. From source file:nl.tudelft.graphalytics.mapreducev2.evo.DirectedForestFireModelMap.java
License:Apache License
@Override public void configure(JobConf conf) { TaskAttemptID attempt = TaskAttemptID.forName(conf.get("mapred.task.id")); this.taskID = attempt.getTaskID().getId(); // todo verify this.newVerticesPerSlot = conf.getInt(ForestFireModelUtils.NEW_VERTICES_NR, -1); this.maxID = conf.getLong(ForestFireModelUtils.MAX_ID, -1); this.isFirst = conf.getBoolean(ForestFireModelUtils.IS_INIT, false); this.isInit = this.isFirst; if (this.isInit) this.ambassadors = new HashMap<LongWritable, List<LongWritable>>(); else/* w w w . j a v a2 s . co m*/ this.ambassadors = ForestFireModelUtils .verticesIdsString2Map(conf.get(ForestFireModelUtils.CURRENT_AMBASSADORS)); }
From source file:nl.tudelft.graphalytics.mapreducev2.evo.UndirectedForestFireModelMap.java
License:Apache License
@Override public void configure(JobConf conf) { TaskAttemptID attempt = TaskAttemptID.forName(conf.get("mapred.task.id")); this.taskID = attempt.getTaskID().getId(); this.newVerticesPerSlot = conf.getInt(ForestFireModelUtils.NEW_VERTICES_NR, -1); this.maxID = conf.getLong(ForestFireModelUtils.MAX_ID, -1); this.isFirst = conf.getBoolean(ForestFireModelUtils.IS_INIT, false); this.isInit = this.isFirst; if (this.isInit) this.ambassadors = new HashMap<LongWritable, List<LongWritable>>(); else/*from w w w . j a va 2 s . c o m*/ this.ambassadors = ForestFireModelUtils .verticesIdsString2Map(conf.get(ForestFireModelUtils.CURRENT_AMBASSADORS)); }
From source file:org.apache.ambari.servicemonitor.jobs.RangeInputFormat.java
License:Apache License
/** * Create the desired number of splits, dividing the number of rows between * the mappers.// w w w .ja v a 2 s. c o m */ public InputSplit[] getSplits(JobConf job, int numSplits) { int totalRows = job.getInt(JobKeys.RANGEINPUTFORMAT_ROWS, 0); int rowsPerSplit = totalRows / numSplits; LOG.info("Generating " + totalRows + " using " + numSplits + " maps with step of " + rowsPerSplit); InputSplit[] splits = new InputSplit[numSplits]; int currentRow = 0; for (int split = 0; split < numSplits - 1; ++split) { splits[split] = new RangeInputSplit(currentRow, rowsPerSplit); currentRow += rowsPerSplit; } splits[numSplits - 1] = new RangeInputSplit(currentRow, totalRows - currentRow); return splits; }
From source file:org.apache.avro.mapred.AvroOutputFormat.java
License:Apache License
static <T> void configureDataFileWriter(DataFileWriter<T> writer, JobConf job) throws UnsupportedEncodingException { CodecFactory factory = getCodecFactory(job); if (factory != null) { writer.setCodec(factory);//from ww w . j ava2s . c o m } writer.setSyncInterval(job.getInt(SYNC_INTERVAL_KEY, DEFAULT_SYNC_INTERVAL)); // copy metadata from job for (Map.Entry<String, String> e : job) { if (e.getKey().startsWith(AvroJob.TEXT_PREFIX)) writer.setMeta(e.getKey().substring(AvroJob.TEXT_PREFIX.length()), e.getValue()); if (e.getKey().startsWith(AvroJob.BINARY_PREFIX)) writer.setMeta(e.getKey().substring(AvroJob.BINARY_PREFIX.length()), URLDecoder.decode(e.getValue(), "ISO-8859-1").getBytes("ISO-8859-1")); } }
From source file:org.apache.avro.mapred.AvroOutputFormat.java
License:Apache License
/** This will select the correct compression codec from the JobConf. * The order of selection is as follows: * <ul>//from www . ja v a 2s .c o m * <li>If mapred.output.compress is true then look for codec otherwise no compression</li> * <li>Use avro.output.codec if populated</li> * <li>Next use mapred.output.compression.codec if populated</li> * <li>If not default to Deflate Codec</li> * </ul> */ static CodecFactory getCodecFactory(JobConf job) { CodecFactory factory = null; if (FileOutputFormat.getCompressOutput(job)) { int level = job.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = job.get(AvroJob.OUTPUT_CODEC); if (codecName == null) { String codecClassName = job.get("mapred.output.compression.codec", null); String avroCodecName = HadoopCodecFactory.getAvroCodecName(codecClassName); if (codecClassName != null && avroCodecName != null) { factory = HadoopCodecFactory.fromHadoopString(codecClassName); job.set(AvroJob.OUTPUT_CODEC, avroCodecName); return factory; } else { return CodecFactory.deflateCodec(level); } } else { if (codecName.equals(DEFLATE_CODEC)) { factory = CodecFactory.deflateCodec(level); } else { factory = CodecFactory.fromString(codecName); } } } return factory; }
From source file:org.apache.avro.mapred.TestAvroOutputFormat.java
License:Apache License
@Test public void testSetSyncInterval() { JobConf jobConf = new JobConf(); int newSyncInterval = 100000; AvroOutputFormat.setSyncInterval(jobConf, newSyncInterval); assertEquals(newSyncInterval, jobConf.getInt(AvroOutputFormat.SYNC_INTERVAL_KEY, -1)); }
From source file:org.apache.avro.mapred.tether.TetherOutputFormat.java
License:Apache License
@SuppressWarnings("unchecked") public RecordWriter<TetherData, NullWritable> getRecordWriter(FileSystem ignore, JobConf job, String name, Progressable prog) throws IOException { Schema schema = AvroJob.getOutputSchema(job); final DataFileWriter writer = new DataFileWriter(new GenericDatumWriter()); if (FileOutputFormat.getCompressOutput(job)) { int level = job.getInt(AvroOutputFormat.DEFLATE_LEVEL_KEY, AvroOutputFormat.DEFAULT_DEFLATE_LEVEL); writer.setCodec(CodecFactory.deflateCodec(level)); }/* ww w . ja v a2 s . c o m*/ Path path = FileOutputFormat.getTaskOutputPath(job, name + AvroOutputFormat.EXT); writer.create(schema, path.getFileSystem(job).create(path)); return new RecordWriter<TetherData, NullWritable>() { public void write(TetherData datum, NullWritable ignore) throws IOException { writer.appendEncoded(datum.buffer()); } public void close(Reporter reporter) throws IOException { writer.close(); } }; }
From source file:org.apache.cassandra.hadoop.ColumnFamilyInputFormat.java
License:Apache License
public org.apache.hadoop.mapred.RecordReader<ByteBuffer, SortedMap<ByteBuffer, ColumnFamilyRecordReader.Column>> getRecordReader( org.apache.hadoop.mapred.InputSplit split, JobConf jobConf, final Reporter reporter) throws IOException { TaskAttemptContext tac = HadoopCompat.newMapContext(jobConf, TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)), null, null, null, new ReporterWrapper(reporter), null);// w ww . j av a 2 s. c o m ColumnFamilyRecordReader recordReader = new ColumnFamilyRecordReader( jobConf.getInt(CASSANDRA_HADOOP_MAX_KEY_SIZE, CASSANDRA_HADOOP_MAX_KEY_SIZE_DEFAULT)); recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac); return recordReader; }
From source file:org.apache.cassandra.hadoop2.ColumnFamilyInputFormat.java
License:Apache License
public org.apache.hadoop.mapred.RecordReader<ByteBuffer, SortedMap<ByteBuffer, Column>> getRecordReader( org.apache.hadoop.mapred.InputSplit split, JobConf jobConf, final Reporter reporter) throws IOException { TaskAttemptContext tac = new TaskAttemptContextImpl(jobConf, TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID))) { @Override/*from ww w. j a v a 2 s . c o m*/ public void progress() { reporter.progress(); } }; ColumnFamilyRecordReader recordReader = new ColumnFamilyRecordReader( jobConf.getInt(CASSANDRA_HADOOP_MAX_KEY_SIZE, CASSANDRA_HADOOP_MAX_KEY_SIZE_DEFAULT)); recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac); return recordReader; }
From source file:org.apache.crunch.types.avro.AvroOutputFormat.java
License:Apache License
@Override public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); Schema schema = null;//from w ww. j a va 2 s .co m String outputName = conf.get("crunch.namedoutput"); if (outputName != null && !outputName.isEmpty()) { schema = (new Schema.Parser()).parse(conf.get("avro.output.schema." + outputName)); } else { schema = AvroJob.getOutputSchema(context.getConfiguration()); } ReflectDataFactory factory = Avros.getReflectDataFactory(conf); final DataFileWriter<T> WRITER = new DataFileWriter<T>(factory.<T>getWriter(schema)); JobConf jc = new JobConf(conf); /* copied from org.apache.avro.mapred.AvroOutputFormat */ if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) { int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY, org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL); String codecName = conf.get(AvroJob.OUTPUT_CODEC, org.apache.avro.file.DataFileConstants.DEFLATE_CODEC); CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); WRITER.setCodec(codec); } WRITER.setSyncInterval(jc.getInt(org.apache.avro.mapred.AvroOutputFormat.SYNC_INTERVAL_KEY, org.apache.avro.file.DataFileConstants.DEFAULT_SYNC_INTERVAL)); Path path = getDefaultWorkFile(context, org.apache.avro.mapred.AvroOutputFormat.EXT); WRITER.create(schema, path.getFileSystem(context.getConfiguration()).create(path)); return new RecordWriter<AvroWrapper<T>, NullWritable>() { @Override public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { WRITER.append(wrapper.datum()); } @Override public void close(TaskAttemptContext context) throws IOException, InterruptedException { WRITER.close(); } }; }