List of usage examples for org.apache.hadoop.mapred JobConf getInt
public int getInt(String name, int defaultValue)
name property as an int. From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.TransEdgeMapper.java
License:Open Source License
@Override public void configure(JobConf conf) { super.configure(conf); numChunks = conf.getInt("numChunks", 256); dictionaryPath = conf.get("dictionaryPath"); dict = new HashMap<VidType, Long>(); dictionaryId = -1;/*from w ww . j ava 2 s . c o m*/ try { fs = FileSystem.get(conf); } catch (IOException e1) { e1.printStackTrace(); } try { this.graphparser = (GraphParser) Class.forName(conf.get("GraphParser")).newInstance(); this.vidparser = (FieldParser) Class.forName(conf.get("VidParser")).newInstance(); this.edataparser = (FieldParser) Class.forName(conf.get("EdataParser")).newInstance(); } catch (InstantiationException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } }
From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.TransEdgeReducer.java
License:Open Source License
@Override public void configure(JobConf conf) { super.configure(conf); numChunks = conf.getInt("numChunks", 256); dictionaryPath = conf.get("dictionaryPath"); dict = new HashMap<VidType, Long>(); dictionaryId = -1;//from w w w . j a v a 2 s . com try { fs = FileSystem.get(conf); } catch (IOException e1) { e1.printStackTrace(); } try { this.graphparser = (GraphParser) Class.forName(conf.get("GraphParser")).newInstance(); this.vidparser = (FieldParser) Class.forName(conf.get("VidParser")).newInstance(); this.edataparser = (FieldParser) Class.forName(conf.get("EdataParser")).newInstance(); } catch (InstantiationException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } }
From source file:com.intel.hadoop.graphbuilder.partition.mapreduce.edge.EdgeIngressMapper.java
License:Open Source License
@SuppressWarnings("unchecked") @Override// ww w . ja v a2s . c o m public void configure(JobConf job) { super.configure(job); this.keyClass = job.getMapOutputKeyClass(); this.valClass = job.getMapOutputValueClass(); numprocs = job.getInt("numProcs", 1); overpartition = job.getInt("overpartition", 1); String ingressMethod = job.get("ingress"); if (ingressMethod.equals("greedy")) { this.ingress = new GreedyIngress<VidType>(numprocs); } else { this.ingress = new RandomIngress<VidType>(numprocs); } try { this.graphparser = (GraphParser) Class.forName(job.get("GraphParser")).newInstance(); this.vidparser = (FieldParser) Class.forName(job.get("VidParser")).newInstance(); this.vdataparser = (FieldParser) Class.forName(job.get("VdataParser")).newInstance(); this.edataparser = (FieldParser) Class.forName(job.get("EdataParser")).newInstance(); this.mapKey = (KeyType) keyClass.newInstance(); this.mapValue = (ValueType) valClass.newInstance(); } catch (InstantiationException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } }
From source file:com.intel.hadoop.graphbuilder.partition.mapreduce.edge.EdgeIngressReducer.java
License:Open Source License
@Override public void configure(JobConf job) { super.configure(job); this.numProcs = job.getInt("numProcs", 1); this.subpartPerPartition = job.getInt("subpartPerPartition", 1); // Switch to GLGraph by uncommenting the next line. // graphOutput = new GLGraphOutput(numProcs); graphOutput = new SimpleGraphOutput(); graphOutput.configure(job);//w w w. j a va 2s.c o m }
From source file:com.jeffy.hive.RowIdUDF.java
License:Apache License
/** * ??context.getJobConf()?jobConfiguration * ?Configuration?/*from w w w. j a v a 2 s. c om*/ * This is only called in runtime of MapRedTask. ??MapReduce?? */ @Override public void configure(MapredContext context) { log.info("Configure RowIdUDF."); if (context != null) { JobConf conf = context.getJobConf(); if (conf == null) { throw new IllegalStateException("JobConf is not set."); } taskId = conf.getInt("mapred.task.partition", -1); if (taskId == -1) { taskId = conf.getInt("mapreduce.task.partition", -1); if (taskId == -1) { throw new IllegalStateException( "Both mapred.task.partition and mapreduce.task.partition are not set: " + conf); } } } }
From source file:com.jeffy.hive.RowIdUDF2.java
License:Apache License
/** * MapredContext??MapReduce???MapReduce/*from w ww.j a v a 2 s .c om*/ * * @return MapredID */ public static int getTaskId() { MapredContext ctx = MapredContext.get(); if (ctx == null) { throw new IllegalStateException("MapredContext is not set."); } JobConf conf = ctx.getJobConf(); if (conf == null) { throw new IllegalStateException("JobConf is not set."); } int taskid = conf.getInt("mapred.task.partition", -1); if (taskid == -1) { taskid = conf.getInt("mapreduce.task.partition", -1); if (taskid == -1) { throw new IllegalStateException( "Both mapred.task.partition and mapreduce.task.partition are not set: " + conf); } } return taskid; }
From source file:com.kadwa.hadoop.DistExec.java
License:Open Source License
/** * Calculate how many maps to run. Number of maps is equal to the number of files. * * @param fileCount Count of total files for job * @param job The job to configure * @return Count of maps to run./*from w w w. ja va 2 s . c o m*/ */ private static void setMapCount(long fileCount, JobConf job) throws IOException { int numMaps = (int) fileCount; numMaps = Math.min(numMaps, job.getInt(MAX_MAPS_LABEL, MAX_MAPS_PER_NODE * new JobClient(job).getClusterStatus().getTaskTrackers())); job.setNumMapTasks(Math.max(numMaps, 1)); }
From source file:com.linkedin.haivvreo.AvroContainerOutputFormat.java
License:Apache License
@Override public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Class<? extends Writable> valueClass, boolean isCompressed, Properties properties, Progressable progressable) throws IOException { Schema schema;//www . j a v a2 s . c o m try { schema = HaivvreoUtils.determineSchemaOrThrowException(properties); } catch (HaivvreoException e) { throw new IOException(e); } GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema); DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw); if (isCompressed) { int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL); String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC); CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName); dfw.setCodec(factory); } dfw.create(schema, path.getFileSystem(jobConf).create(path)); return new AvroGenericRecordWriter(dfw); }
From source file:com.pinterest.hdfsbackup.distcp.DistCp.java
License:Apache License
/** * Calculate how many maps to run.// www . ja v a2s. co m * Number of maps is bounded by a minimum of the cumulative size of the * copy / (distcp.bytes.per.map, default BYTES_PER_MAP or -m on the * command line) and at most (distcp.max.map.tasks, default * MAX_MAPS_PER_NODE * nodes in the cluster). * @param totalBytes Count of total bytes for job * @param job The job to configure * @return Count of maps to run. */ private static void setMapCount(long totalBytes, JobConf job) throws IOException { int numMaps = (int) (totalBytes / job.getLong(BYTES_PER_MAP_LABEL, BYTES_PER_MAP)); numMaps = Math.min(numMaps, job.getInt(MAX_MAPS_LABEL, MAX_MAPS_PER_NODE * new JobClient(job).getClusterStatus().getTaskTrackers())); job.setNumMapTasks(Math.max(numMaps, 1)); }
From source file:com.scaleoutsoftware.soss.hserver.NamedMapInputFormatMapred.java
License:Apache License
@Override public InputSplit[] getSplits(JobConf configuration, int i) throws IOException { int mapId = configuration.getInt(inputAppIdProperty, 0); if (mapId == 0) { throw new IOException("Input format is not configured with a valid NamedMap."); }/*from ww w. j a va 2 s .com*/ List<org.apache.hadoop.mapreduce.InputSplit> splits; try { splits = GridInputFormat.getSplits(mapId, i); } catch (InterruptedException e) { throw new IOException(e); } InputSplit[] wrappedSpilts = new InputSplit[splits.size()]; Iterator splitIterator = splits.iterator(); //Wrap splits to conform to mapred API for (i = 0; i < wrappedSpilts.length; i++) { wrappedSpilts[i] = new BucketSplitMapred((BucketSplit) splitIterator.next()); } return wrappedSpilts; }