Example usage for org.apache.hadoop.mapreduce TaskInputOutputContext progress

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskInputOutputContext progress.

Prototype

public void progress();

Source Link

Document

Report progress to the Hadoop framework.

Usage

From source file:com.cloudera.oryx.als.computation.iterate.row.YState.java

License:Open Source License

synchronized void initialize(TaskInputOutputContext<?, ?, ?, ?> context, int currentPartition,
        int numPartitions) {

    Configuration conf = context.getConfiguration();

    LongSet expectedIDs;//from  w  ww .  j av a  2 s.  co  m
    try {
        expectedIDs = ComputationDataUtils.readExpectedIDsFromPartition(currentPartition, numPartitions,
                conf.get(RowStep.POPULAR_KEY), context, conf);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }

    String yKey = conf.get(RowStep.Y_KEY_KEY);
    log.info("Reading X or Y from {}", yKey);

    Y = new LongObjectMap<float[]>();

    Iterable<MatrixRow> in;
    try {
        in = new AvroFileSource<MatrixRow>(Namespaces.toPath(yKey), (AvroType<MatrixRow>) ptype).read(conf);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }

    RealMatrix theYTY = null;
    int dimension = 0;
    long count = 0;
    for (MatrixRow record : in) {
        long keyID = record.getRowId();
        float[] vector = record.getValues();
        Preconditions.checkNotNull(vector, "Vector was null for %s?", keyID);

        if (theYTY == null) {
            dimension = vector.length;
            theYTY = new Array2DRowRealMatrix(dimension, dimension);
        }
        for (int row = 0; row < dimension; row++) {
            double rowValue = vector[row];
            for (int col = 0; col < dimension; col++) {
                theYTY.addToEntry(row, col, rowValue * vector[col]);
            }
        }

        if (expectedIDs == null || expectedIDs.contains(keyID)) {
            Y.put(keyID, vector);
        }

        if (++count % 1000 == 0) {
            context.progress();
        }
    }

    Preconditions.checkNotNull(theYTY);
    YTY = theYTY;
}

---