Example usage for org.apache.hadoop.io DefaultStringifier loadArray

Introduction

In this page you can find the example usage for org.apache.hadoop.io DefaultStringifier loadArray.

Prototype

public static <K> K[] loadArray(Configuration conf, String keyName, Class<K> itemClass) throws IOException 

Source Link

Document

Restores the array of objects from the configuration.

Usage

From source file:RandIntPartSamplerMapper.java

License:Apache License

@Override
public void configure(JobConf conf) {
    id = conf.getInt("mapred.task.partition", -1);
    reducersNum = conf.getInt("PARMM.reducersNum", 1000);
    try {//  ww  w  . j a  va  2  s  . c  o m
        int id = conf.getInt("mapred.task.partition", -1);
        System.out.println("id: " + id);
        IntWritable[] toSampleArr = DefaultStringifier.loadArray(conf, "PARMM.toSampleArr_" + id,
                IntWritable.class);
        toSample = 0;
        for (IntWritable toSampleRed : toSampleArr) {
            toSample += toSampleRed.get();
        }
        System.out.println("toSample: " + toSample);
        sampleDestinations = new IntWritable[toSample];
        int i = 0;
        for (int k = 0; k < toSampleArr.length; k++) {
            for (int j = 0; j < toSampleArr[k].get(); j++) {
                sampleDestinations[i++] = new IntWritable(k);
            }
        }
        Collections.shuffle(Arrays.asList(sampleDestinations));
    } catch (IOException e) {
    }
}

---

From source file:org.apache.sqoop.mapreduce.hcat.SqoopHCatImportHelper.java

License:Apache License

public SqoopHCatImportHelper(Configuration conf) throws IOException, InterruptedException {

    String inputJobInfoStr = conf.get(HCatConstants.HCAT_KEY_JOB_INFO);
    jobInfo = (InputJobInfo) HCatUtil.deserialize(inputJobInfoStr);
    dataColsSchema = jobInfo.getTableInfo().getDataColumns();
    partitionSchema = jobInfo.getTableInfo().getPartitionColumns();
    StringBuilder storerInfoStr = new StringBuilder(1024);
    StorerInfo storerInfo = jobInfo.getTableInfo().getStorerInfo();
    storerInfoStr.append("HCatalog Storer Info : ").append("\n\tHandler = ")
            .append(storerInfo.getStorageHandlerClass()).append("\n\tInput format class = ")
            .append(storerInfo.getIfClass()).append("\n\tOutput format class = ")
            .append(storerInfo.getOfClass()).append("\n\tSerde class = ").append(storerInfo.getSerdeClass());
    Properties storerProperties = storerInfo.getProperties();
    if (!storerProperties.isEmpty()) {
        storerInfoStr.append("\nStorer properties ");
        for (Map.Entry<Object, Object> entry : storerProperties.entrySet()) {
            String key = (String) entry.getKey();
            Object val = entry.getValue();
            storerInfoStr.append("\n\t").append(key).append('=').append(val);
        }//from   w  ww.  ja  v  a 2 s  . c o m
    }
    storerInfoStr.append("\n");
    LOG.info(storerInfoStr);

    hCatFullTableSchema = new HCatSchema(dataColsSchema.getFields());
    for (HCatFieldSchema hfs : partitionSchema.getFields()) {
        hCatFullTableSchema.append(hfs);
    }
    fieldCount = hCatFullTableSchema.size();
    lobLoader = new LargeObjectLoader(conf, new Path(jobInfo.getTableInfo().getTableLocation()));
    bigDecimalFormatString = conf.getBoolean(ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
            ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
    debugHCatImportMapper = conf.getBoolean(SqoopHCatUtilities.DEBUG_HCAT_IMPORT_MAPPER_PROP, false);
    IntWritable[] delimChars = DefaultStringifier.loadArray(conf,
            SqoopHCatUtilities.HIVE_DELIMITERS_TO_REPLACE_PROP, IntWritable.class);
    hiveDelimiters = new DelimiterSet((char) delimChars[0].get(), (char) delimChars[1].get(),
            (char) delimChars[2].get(), (char) delimChars[3].get(), delimChars[4].get() == 1 ? true : false);
    hiveDelimsReplacement = conf.get(SqoopHCatUtilities.HIVE_DELIMITERS_REPLACEMENT_PROP);
    if (hiveDelimsReplacement == null) {
        hiveDelimsReplacement = "";
    }
    doHiveDelimsReplacement = Boolean
            .valueOf(conf.get(SqoopHCatUtilities.HIVE_DELIMITERS_REPLACEMENT_ENABLED_PROP));

    IntWritable[] fPos = DefaultStringifier.loadArray(conf, SqoopHCatUtilities.HCAT_FIELD_POSITIONS_PROP,
            IntWritable.class);
    hCatFieldPositions = new int[fPos.length];
    for (int i = 0; i < fPos.length; ++i) {
        hCatFieldPositions[i] = fPos[i].get();
    }

    LOG.debug("Hive delims replacement enabled : " + doHiveDelimsReplacement);
    LOG.debug("Hive Delimiters : " + hiveDelimiters.toString());
    LOG.debug("Hive delimiters replacement : " + hiveDelimsReplacement);
    staticPartitionKey = conf.get(SqoopHCatUtilities.HCAT_STATIC_PARTITION_KEY_PROP);
    LOG.debug("Static partition key used : " + staticPartitionKey);
}

---