List of usage examples for org.apache.hadoop.mapred JobConf getClassByName
public Class<?> getClassByName(String name) throws ClassNotFoundException
From source file:cascading.tap.hadoop.io.MultiInputSplit.java
License:Open Source License
public void readFields(DataInput in) throws IOException { String splitType = in.readUTF(); config = new HashMap<String, String>(); String[] keys = WritableUtils.readStringArray(in); String[] values = WritableUtils.readStringArray(in); for (int i = 0; i < keys.length; i++) config.put(keys[i], values[i]);/* w w w . ja v a 2s .co m*/ if (LOG.isDebugEnabled()) { LOG.debug("current split config diff:"); for (Map.Entry<String, String> entry : config.entrySet()) LOG.debug("key: {}, value: {}", entry.getKey(), entry.getValue()); } JobConf currentConf = HadoopUtil.mergeConf(jobConf, config, false); try { inputSplit = (InputSplit) ReflectionUtils.newInstance(currentConf.getClassByName(splitType), currentConf); } catch (ClassNotFoundException exp) { throw new IOException("split class " + splitType + " not found"); } inputSplit.readFields(in); if (inputSplit instanceof FileSplit) { Path path = ((FileSplit) inputSplit).getPath(); if (path != null) { jobConf.set(CASCADING_SOURCE_PATH, path.toString()); LOG.info("current split input path: {}", path); } } }
From source file:cascading.tap.hadoop.MultiInputSplit.java
License:Open Source License
public void readFields(DataInput in) throws IOException { String splitType = in.readUTF(); config = new HashMap<String, String>(); String[] keys = WritableUtils.readStringArray(in); String[] values = WritableUtils.readStringArray(in); for (int i = 0; i < keys.length; i++) config.put(keys[i], values[i]);/*from w w w. jav a 2 s. c o m*/ JobConf currentConf = MultiInputFormat.mergeConf(jobConf, config, false); try { inputSplit = (InputSplit) ReflectionUtils.newInstance(currentConf.getClassByName(splitType), currentConf); } catch (ClassNotFoundException exp) { throw new IOException("split class " + splitType + " not found"); } inputSplit.readFields(in); if (inputSplit instanceof FileSplit) { Path path = ((FileSplit) inputSplit).getPath(); if (path != null) { jobConf.set("cascading.source.path", path.toString()); if (LOG.isInfoEnabled()) LOG.info("current split input path: " + path.toString()); } } }
From source file:com.cloudera.recordservice.hive.RecordServiceHiveInputFormat.java
License:Apache License
/** * Replaces whatever the actual InputFormat/RecordReader is with the RecordService * version./*w w w.j a va2 s .co m*/ */ @Override public RecordReader<K, V> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { // Initialize everything needed to read the projection information. HiveInputSplit hsplit = (HiveInputSplit) split; if (pathToPartitionInfo == null) init(job); PartitionDesc part = pathToPartitionInfo.get(hsplit.getPath().toString()); if ((part != null) && (part.getTableDesc() != null)) { Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), job); } // Create a RecordServiceRecordReader and wrap it with a HiveRecordReader // to read the data. RecordServiceRecordReader rsRr = new RecordServiceRecordReader( (RecordServiceInputSplit) hsplit.getInputSplit(), job, reporter); // Pass the RecordService as the target InputFormat (this will override // the actual input format). String inputFormatClassName = RecordServiceInputFormat.class.getName(); Class cl; try { cl = job.getClassByName(inputFormatClassName); } catch (ClassNotFoundException e) { throw new IOException("cannot find class " + inputFormatClassName, e); } HiveRecordReaderShim<K, V> rr = new HiveRecordReaderShim<K, V>(rsRr, job); // Initialize the Hive RecordReader IO Context so that it will do the proper thing. rr.initIOContext(hsplit, job, cl); return rr; }
From source file:com.facebook.presto.hive.HiveUtil.java
License:Apache License
@SuppressWarnings({ "unchecked", "RedundantCast" })
private static Class<? extends InputFormat<?, ?>> getInputFormatClass(JobConf conf, String inputFormatName)
throws ClassNotFoundException {
Class<?> clazz = conf.getClassByName(inputFormatName);
// TODO: remove redundant cast to Object after IDEA-118533 is fixed
return (Class<? extends InputFormat<?, ?>>) (Object) clazz.asSubclass(InputFormat.class);
}
From source file:com.facebook.presto.hive.HiveWriterFactory.java
License:Apache License
public static String getFileExtension(JobConf conf, StorageFormat storageFormat) { // text format files must have the correct extension when compressed if (!HiveConf.getBoolVar(conf, COMPRESSRESULT) || !HiveIgnoreKeyTextOutputFormat.class.getName().equals(storageFormat.getOutputFormat())) { return ""; }/*from w w w .j a va 2s . c o m*/ String compressionCodecClass = conf.get("mapred.output.compression.codec"); if (compressionCodecClass == null) { return new DefaultCodec().getDefaultExtension(); } try { Class<? extends CompressionCodec> codecClass = conf.getClassByName(compressionCodecClass) .asSubclass(CompressionCodec.class); return ReflectionUtil.newInstance(codecClass, conf).getDefaultExtension(); } catch (ClassNotFoundException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Compression codec not found: " + compressionCodecClass, e); } catch (RuntimeException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Failed to load compression codec: " + compressionCodecClass, e); } }
From source file:com.ibm.bi.dml.runtime.matrix.data.hadoopfix.MultipleInputs.java
License:Apache License
/** * Retrieves a map of {@link Path}s to the {@link InputFormat} class * that should be used for them.//from w w w . j a va 2 s . co m * * @param conf The confuration of the job * @see #addInputPath(JobConf, Path, Class) * @return A map of paths to inputformats for the job */ static Map<Path, InputFormat> getInputFormatMap(JobConf conf) { Map<Path, InputFormat> m = new HashMap<Path, InputFormat>(); String[] pathMappings = conf.get("mapred.input.dir.formats").split(","); for (String pathMapping : pathMappings) { String[] split = pathMapping.split(";"); InputFormat inputFormat; try { inputFormat = (InputFormat) ReflectionUtils.newInstance(conf.getClassByName(split[1]), conf); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } m.put(new Path(split[0]), inputFormat); } return m; }
From source file:com.ibm.bi.dml.runtime.matrix.data.hadoopfix.MultipleInputs.java
License:Apache License
/** * Retrieves a map of {@link Path}s to the {@link Mapper} class that * should be used for them.//from w w w. j a v a 2s. com * * @param conf The confuration of the job * @see #addInputPath(JobConf, Path, Class, Class) * @return A map of paths to mappers for the job */ @SuppressWarnings("unchecked") static Map<Path, Class<? extends Mapper>> getMapperTypeMap(JobConf conf) { if (conf.get("mapred.input.dir.mappers") == null) { return Collections.emptyMap(); } Map<Path, Class<? extends Mapper>> m = new HashMap<Path, Class<? extends Mapper>>(); String[] pathMappings = conf.get("mapred.input.dir.mappers").split(","); for (String pathMapping : pathMappings) { String[] split = pathMapping.split(";"); Class<? extends Mapper> mapClass; try { mapClass = (Class<? extends Mapper>) conf.getClassByName(split[1]); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } m.put(new Path(split[0]), mapClass); } return m; }
From source file:io.prestosql.plugin.hive.HiveUtil.java
License:Apache License
@SuppressWarnings({ "unchecked", "RedundantCast" })
private static Class<? extends InputFormat<?, ?>> getInputFormatClass(JobConf conf, String inputFormatName)
throws ClassNotFoundException {
// CDH uses different names for Parquet
if ("parquet.hive.DeprecatedParquetInputFormat".equals(inputFormatName)
|| "parquet.hive.MapredParquetInputFormat".equals(inputFormatName)) {
return MapredParquetInputFormat.class;
}/*from ww w . j av a2s . co m*/
Class<?> clazz = conf.getClassByName(inputFormatName);
return (Class<? extends InputFormat<?, ?>>) clazz.asSubclass(InputFormat.class);
}
From source file:it.crs4.pydoop.pipes.Submitter.java
License:Apache License
private static <InterfaceType> Class<? extends InterfaceType> getClass(CommandLine cl, String key, JobConf conf, Class<InterfaceType> cls) throws ClassNotFoundException { return conf.getClassByName(cl.getOptionValue(key)).asSubclass(cls); }
From source file:org.apache.avro.mapred.AvroMultipleInputs.java
License:Apache License
/** * Retrieves a map of {@link Path}s to the {@link AvroMapper} class that * should be used for them.//from ww w . j ava 2s . c o m * * @param conf The configuration of the job * @see #addInputPath(JobConf, Path, Class, Schema) * @return A map of paths-to-mappers for the job */ @SuppressWarnings("unchecked") static Map<Path, Class<? extends AvroMapper>> getMapperTypeMap(JobConf conf) { if (conf.get(mappersKey) == null) { return Collections.emptyMap(); } Map<Path, Class<? extends AvroMapper>> m = new HashMap<Path, Class<? extends AvroMapper>>(); String[] pathMappings = conf.get(mappersKey).split(","); for (String pathMapping : pathMappings) { String[] split = pathMapping.split(";"); Class<? extends AvroMapper> mapClass; try { mapClass = (Class<? extends AvroMapper>) conf.getClassByName(split[1]); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } m.put(new Path(split[0]), mapClass); } return m; }