Example usage for org.apache.hadoop.mapred JobConf getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf getInt.

Prototype

public int getInt(String name, int defaultValue) 

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static MatrixCharacteristics getMatrixCharacteristicsForOutput(JobConf job, byte tag) {
    MatrixCharacteristics dim = new MatrixCharacteristics();
    dim.setDimension(job.getLong(OUTPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, 0),
            job.getLong(OUTPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, 0));
    dim.setBlockSize(job.getInt(OUTPUT_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, 1),
            job.getInt(OUTPUT_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, 1));
    return dim;//from   ww w  .ja  v a 2 s .com
}

---

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static MatrixCharacteristics getMatrixCharacteristicsForInput(JobConf job, byte tag) {
    MatrixCharacteristics dim = new MatrixCharacteristics();
    dim.setDimension(job.getLong(INPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, 0),
            job.getLong(INPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, 0));
    dim.setBlockSize(job.getInt(INPUT_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, 1),
            job.getInt(INPUT_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, 1));
    return dim;/*from   www . java 2  s .  c  o  m*/
}

---

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static MatrixCharacteristics getMatrixCharacteristicsForMapOutput(JobConf job, byte tag) {
    MatrixCharacteristics dim = new MatrixCharacteristics();
    dim.setDimension(job.getLong(MAPOUTPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, 0),
            job.getLong(MAPOUTPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, 0));
    dim.setBlockSize(job.getInt(MAPOUTPUT_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, 1),
            job.getInt(MAPOUTPUT_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, 1));
    return dim;/*from   w  ww . j  a  v  a 2s. c  om*/
}

---

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static MatrixCharacteristics getMatrixCharactristicsForReblock(JobConf job, byte tag) {
    MatrixCharacteristics dim = new MatrixCharacteristics();
    dim.setDimension(job.getLong(REBLOCK_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, 0),
            job.getLong(REBLOCK_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, 0));
    dim.setBlockSize(job.getInt(REBLOCK_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, 1),
            job.getInt(REBLOCK_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, 1));

    long nnz = job.getLong(REBLOCK_MATRIX_NUM_NNZ_PREFIX_CONFIG + tag, -1);
    if (nnz >= 0)
        dim.setNonZeros(nnz);//  www  .ja va 2  s  . c  o m

    return dim;
}

---

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static MatrixCharacteristics getMatrixCharactristicsForBinAgg(JobConf job, byte tag) {
    MatrixCharacteristics dim = new MatrixCharacteristics();
    dim.setDimension(job.getLong(AGGBIN_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, 0),
            job.getLong(AGGBIN_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, 0));
    dim.setBlockSize(job.getInt(AGGBIN_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, 1),
            job.getInt(AGGBIN_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, 1));
    return dim;//  w  w  w .jav  a2  s. c  om
}

---

From source file:com.ibm.jaql.fail.io.ErrorInputFormat.java

License:Apache License

@Override
public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException {
    String val = arg0.get(ERROR_NAME);
    Error err = Error.valueOf(val);
    // fail on SPLIT,
    if (err.equals(Error.SPLIT)) {
        throw new IOException("Intentional error on split");
    }/* w  w  w  .j  a v a  2 s  .  co m*/

    int nextCount = arg0.getInt(ERROR_NEXT_MAX, 1);

    // get the original splits
    InputSplit[] splits = super.getSplits(arg0, arg1);
    int len = splits.length;
    ArrayList<InputSplit> newSplits = new ArrayList<InputSplit>(len + 1);
    for (int i = 0; i < len; i++) {
        newSplits.add(new ErrorSplit((FileSplit) splits[i], arg0, err, nextCount));
    }
    // generate bogus split for BOGUS_SPLIT
    if (err.equals(Error.BOGUS_SPLIT)) {

        newSplits.add(new ErrorSplit(new FileSplit(new Path("/bogus/file"), 0, 10, arg0), arg0, Error.NONE,
                nextCount));
    }

    return newSplits.toArray(new InputSplit[newSplits.size()]);
}

---

From source file:com.ibm.jaql.fail.io.ErrorOutputFormat.java

License:Apache License

@Override
public RecordWriter<LongWritable, ErrorWritable> getRecordWriter(FileSystem arg0, JobConf arg1, String arg2,
        Progressable arg3) throws IOException {
    String val = arg1.get(ERROR_NAME);
    Error e = Error.valueOf(val);
    // if OPEN, fail
    if (e.equals(Error.OPEN)) {
        throw new IOException("Intentional error on open");
    }//  w w w  . j a  v a  2  s. co  m
    int max = arg1.getInt(ERROR_NEXT_MAX, 1);
    // Wrap in an ErrorRecordWriter
    return new ErrorRecordWriter(super.getRecordWriter(arg0, arg1, arg2, arg3), e, max);
}

---

From source file:com.ibm.jaql.lang.expr.system.RJaqlInterface.java

License:Apache License

/**
 * This method provides the functionality of saving simple R objects into HDFS in one of
 * the formats supported by Jaql so that it can be directly read into Jaql.
 * @param localPath//from  w  w  w  .  j  a  v a2s  . co m
 * @param hdfsPath
 * @param schemaString
 * @param format
 * @param header
 * @param vector
 * @return
 */
public boolean jaqlSave(String localPath, String hdfsPath, String schemaString, String format, boolean header,
        boolean vector) {
    if (format.equalsIgnoreCase(FORMAT_DELIM)) {
        LOG.info("Format: " + FORMAT_DELIM + ", saving to HDFS loc: " + hdfsPath);
        return RUtil.saveToHDFS(localPath, hdfsPath);
    }
    try {
        JobConf conf = new JobConf();
        int DEFAULT_BUFFER_SIZE = 64 * 1024;
        int bufferSize = conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE);
        BufferedReader reader = new BufferedReader(new FileReader(localPath), bufferSize);
        LongWritable key = new LongWritable(0);
        long count = 0;
        Text value = new Text();
        BufferedJsonRecord options = new BufferedJsonRecord(2);
        BufferedJsonArray headerArray = null;
        if (header) {
            String headerString = reader.readLine();
            String[] headers = splitPattern.split(headerString);
            headerArray = new BufferedJsonArray(headers.length);
            for (int i = 0; i < headers.length; i++) {
                headerArray.set(i, new JsonString(StringUtils.strip(headers[i], "\"")));
            }
            count++;
        }

        Schema schema = null;
        if (schemaString != null) {
            schema = SchemaFactory.parse(schemaString);
        }

        if (headerArray != null) {
            RecordSchema recordSchema = (RecordSchema) schema;

            // construct new matching schema
            List<Field> fields = new LinkedList<Field>();
            for (JsonValue fieldName : headerArray) {
                Field field;
                if (recordSchema == null) {
                    field = new Field((JsonString) fieldName, SchemaFactory.stringSchema(), false);
                } else {
                    field = recordSchema.getField((JsonString) fieldName);
                    if (field == null)
                        throw new NullPointerException("header field not in schema: " + fieldName);
                    // FIXME: schema fields that are not in the header are currently consider OK
                }
                fields.add(field);
            }

            // and set it
            schema = new RecordSchema(fields, null);
        }
        if (schema != null)
            options.add(DelOptionParser.SCHEMA_NAME, new JsonSchema(schema));
        KeyValueImport<LongWritable, Text> converter = null;
        if (vector) {
            converter = new FromLinesConverter();
        } else {
            converter = new FromDelConverter();
        }
        LOG.info("Initializing Converter with options: " + options);
        converter.init(options);
        Schema tmpSchema = converter.getSchema();
        tmpSchema = SchemaTransformation.removeNullability(tmpSchema);
        if (!tmpSchema.is(JsonType.ARRAY, JsonType.RECORD, JsonType.BOOLEAN, JsonType.DECFLOAT, JsonType.DOUBLE,
                JsonType.LONG, JsonType.STRING).always()) {
            throw new IOException("Unrecognized schema type: " + schema.getSchemaType());
        }
        JsonValue outValue = converter.createTarget();
        JsonHolder outKeyHolder;
        JsonHolder outValueHolder;
        if (format.equalsIgnoreCase(FORMAT_DEFAULT)) {
            HadoopSerializationDefault.register(conf);
            outKeyHolder = new JsonHolderDefault();
            outValueHolder = new JsonHolderDefault(outValue);
            LOG.info("Registered serializer for Default format.");
        } else if (format.equalsIgnoreCase(FORMAT_TEMP)) {
            // TODO: There should be a better way of doing this. HadoopSerializationTemp
            // now does it in an ugly way.
            BufferedJsonRecord tmpOptions = new BufferedJsonRecord();
            BufferedJsonRecord outOptions = new BufferedJsonRecord();
            outOptions.add(new JsonString("schema"), new JsonSchema(schema));
            tmpOptions.add(new JsonString("options"), outOptions);
            conf.set(ConfSetter.CONFOUTOPTIONS_NAME, tmpOptions.toString());
            HadoopSerializationTemp.register(conf);
            outKeyHolder = new JsonHolderTempKey(null);
            outValueHolder = new JsonHolderTempValue();
            LOG.info("Registered serializer for HadoopTemp format.");
        } else {
            throw new IOException("Unrecognized serialization format requested: " + format);
        }
        FileSystem fs = FileSystem.get(conf);
        Path outputPath = new Path(hdfsPath);
        Writer writer = SequenceFile.createWriter(fs, conf, outputPath, outKeyHolder.getClass(),
                outValueHolder.getClass());
        String line;
        while ((line = reader.readLine()) != null) {
            key.set(count++);
            value.set(line);
            outValue = converter.convert(key, value, outValue);
            outValueHolder.value = outValue;
            writer.append(outKeyHolder, outValueHolder);
        }
        LOG.info("Transferred " + count + " line(s).");
        reader.close();
        writer.close();
    } catch (IOException e) {
        LOG.info("Error in saving object.", e);
        return false;
    }
    return true;
}

---

From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.HashIdReducer.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    splitsize = job.getInt("mapred.line.input.format.linespermap", 600000);
    try {//from   w  ww  .  j a  v  a2s  . c  o m
        this.graphparser = (GraphParser) Class.forName(job.get("GraphParser")).newInstance();
        this.vidparser = (FieldParser) Class.forName(job.get("VidParser")).newInstance();
        this.vdataparser = (FieldParser) Class.forName(job.get("VdataParser")).newInstance();
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
}

---

From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.SortDictMapper.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    this.hashRawVid = job.getBoolean("hashRawVid", true);
    this.numChunks = job.getInt("numChunks", 256);
    try {//from w  w  w.j av  a 2s.com
        this.vidparser = (FieldParser) Class.forName(job.get("VidParser")).newInstance();
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• »