Example usage for java.util.concurrent ExecutorService invokeAll

List of usage examples for java.util.concurrent ExecutorService invokeAll

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. ExecutorService
  6. invokeAll

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorService invokeAll.

Prototype

<T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException;

Source Link

Document

Executes the given tasks, returning a list of Futures holding their status and results when all complete.

Usage

From source file:com.ibm.bi.dml.runtime.matrix.data.LibMatrixMult.java

/**
 * //from ww  w .ja  v  a  2s.c  om
 * @param mX
 * @param mU
 * @param mV
 * @param mW
 * @param ret
 * @param wt
 * @throws DMLRuntimeException 
 */
public static void matrixMultWSLoss(MatrixBlock mX, MatrixBlock mU, MatrixBlock mV, MatrixBlock mW,
        MatrixBlock ret, WeightsType wt, int k) throws DMLRuntimeException {
    //check for empty result
    if (wt == WeightsType.POST && mW.isEmptyBlock(false)
            || wt == WeightsType.POST_NZ && mX.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;
    }

    //check no parallelization benefit (fallback to sequential)
    if (mX.rlen == 1) {
        matrixMultWSLoss(mX, mU, mV, mW, ret, wt);
        return;
    }

    //Timing time = new Timing(true);

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<ScalarResultTask> tasks = new ArrayList<ScalarResultTask>();
        int blklen = (int) (Math.ceil((double) mX.rlen / k));
        for (int i = 0; i < k & i * blklen < mX.rlen; i++)
            tasks.add(new MatrixMultWSLossTask(mX, mU, mV, mW, wt, i * blklen,
                    Math.min((i + 1) * blklen, mX.rlen)));
        pool.invokeAll(tasks);
        pool.shutdown();
        //aggregate partial results
        sumScalarResults(tasks, ret);
    } catch (InterruptedException e) {
        throw new DMLRuntimeException(e);
    }

    //System.out.println("MMWSLoss "+wt.toString()+" k="+k+" ("+mX.isInSparseFormat()+","+mX.getNumRows()+","+mX.getNumColumns()+","+mX.getNonZeros()+")x" +
    //                   "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop());
}

From source file:com.ibm.bi.dml.runtime.matrix.data.LibMatrixMult.java

/**
 * //from  w  ww.  j a  v  a 2 s . c  o  m
 * @param m1
 * @param m2
 * @param ret1
 * @param ret2
 * @throws DMLUnsupportedOperationException
 * @throws DMLRuntimeException
 * @throws DMLRuntimeException 
 */
public static void matrixMultPermute(MatrixBlock pm1, MatrixBlock m2, MatrixBlock ret1, MatrixBlock ret2, int k)
        throws DMLUnsupportedOperationException, DMLRuntimeException {
    //check inputs / outputs
    if (pm1.isEmptyBlock(false) || m2.isEmptyBlock(false))
        return;

    //check no parallelization benefit (fallback to sequential)
    if (pm1.rlen == 1) {
        matrixMultPermute(pm1, m2, ret1, ret2);
        return;
    }

    //Timing time = new Timing(true);

    //allocate first output block (second allocated if needed)
    ret1.sparse = false;
    ret1.allocateDenseBlock();

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultPermuteTask> tasks = new ArrayList<MatrixMultPermuteTask>();
        int blklen = (int) (Math.ceil((double) pm1.rlen / k));
        for (int i = 0; i < k & i * blklen < pm1.rlen; i++)
            tasks.add(new MatrixMultPermuteTask(pm1, m2, ret1, ret2, i * blklen,
                    Math.min((i + 1) * blklen, pm1.rlen)));
        pool.invokeAll(tasks);
        pool.shutdown();
    } catch (InterruptedException e) {
        throw new DMLRuntimeException(e);
    }

    //post-processing
    ret1.recomputeNonZeros();
    ret1.examSparsity();
    if (ret2 != null) { //optional second output
        ret2.recomputeNonZeros();
        ret2.examSparsity();
    }

    // System.out.println("PMM Par ("+pm1.isInSparseFormat()+","+pm1.getNumRows()+","+pm1.getNumColumns()+","+pm1.getNonZeros()+")x" +
    //                   "("+m2.isInSparseFormat()+","+m2.getNumRows()+","+m2.getNumColumns()+","+m2.getNonZeros()+") in "+time.stop());
}

From source file:com.ibm.bi.dml.runtime.matrix.data.LibMatrixMult.java

/**
 * /*from w w  w .ja  va 2 s  . c  om*/
 * @param mW
 * @param mU
 * @param mV
 * @param ret
 * @param wt
 * @param k
 * @throws DMLRuntimeException
 */
public static void matrixMultWSigmoid(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, MatrixBlock ret,
        WSigmoidType wt, int k) throws DMLRuntimeException {
    //check for empty result
    if (mW.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;
    }

    //check no parallelization benefit (fallback to sequential)
    if (mW.rlen == 1) {
        matrixMultWSigmoid(mW, mU, mV, ret, wt);
        return;
    }

    //Timing time = new Timing(true);

    //pre-processing
    ret.sparse = mW.sparse;
    ret.allocateDenseOrSparseBlock();

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultWSigmoidTask> tasks = new ArrayList<MatrixMultWSigmoidTask>();
        int blklen = (int) (Math.ceil((double) mW.rlen / k));
        for (int i = 0; i < k & i * blklen < mW.rlen; i++)
            tasks.add(new MatrixMultWSigmoidTask(mW, mU, mV, ret, wt, i * blklen,
                    Math.min((i + 1) * blklen, mW.rlen)));
        pool.invokeAll(tasks);
        pool.shutdown();
        ret.nonZeros = 0; //reset after execute
        for (MatrixMultWSigmoidTask task : tasks)
            ret.nonZeros += task.getPartialNnz();
    } catch (InterruptedException e) {
        throw new DMLRuntimeException(e);
    }

    //post-processing (nnz maintained in parallel)
    ret.examSparsity();

    //System.out.println("MMWSig "+wt.toString()+" k="+k+" ("+mW.isInSparseFormat()+","+mW.getNumRows()+","+mW.getNumColumns()+","+mW.getNonZeros()+")x" +
    //                   "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop() + ".");
}

From source file:org.apache.sysml.runtime.matrix.data.LibMatrixMult.java

public static void matrixMultPermute(MatrixBlock pm1, MatrixBlock m2, MatrixBlock ret1, MatrixBlock ret2, int k)
        throws DMLRuntimeException {
    //check inputs / outputs
    if (pm1.isEmptyBlock(false) || m2.isEmptyBlock(false))
        return;/*  www . j  a  va 2 s .  c  o m*/

    //check no parallelization benefit (fallback to sequential)
    if (pm1.rlen == 1) {
        matrixMultPermute(pm1, m2, ret1, ret2);
        return;
    }

    //Timing time = new Timing(true);

    //allocate first output block (second allocated if needed)
    ret1.sparse = false; // no need to check isThreadSafe
    ret1.allocateDenseBlock();

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultPermuteTask> tasks = new ArrayList<MatrixMultPermuteTask>();
        int blklen = (int) (Math.ceil((double) pm1.rlen / k));
        for (int i = 0; i < k & i * blklen < pm1.rlen; i++)
            tasks.add(new MatrixMultPermuteTask(pm1, m2, ret1, ret2, i * blklen,
                    Math.min((i + 1) * blklen, pm1.rlen)));
        pool.invokeAll(tasks);
        pool.shutdown();
    } catch (InterruptedException e) {
        throw new DMLRuntimeException(e);
    }

    //post-processing
    ret1.recomputeNonZeros();
    ret1.examSparsity();
    if (ret2 != null) { //optional second output
        ret2.recomputeNonZeros();
        ret2.examSparsity();
    }

    // System.out.println("PMM Par ("+pm1.isInSparseFormat()+","+pm1.getNumRows()+","+pm1.getNumColumns()+","+pm1.getNonZeros()+")x" +
    //                   "("+m2.isInSparseFormat()+","+m2.getNumRows()+","+m2.getNumColumns()+","+m2.getNonZeros()+") in "+time.stop());
}

From source file:org.apache.sysml.runtime.matrix.data.LibMatrixMult.java

public static void matrixMultWCeMM(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, double eps, MatrixBlock ret,
        WCeMMType wt, int k) throws DMLRuntimeException {
    //check for empty result 
    if (mW.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;/*from  w ww  .  jav a2  s  .  c  o  m*/
    }

    //Timing time = new Timing(true);

    //pre-processing (no need to check isThreadSafe)
    ret.sparse = false;
    ret.allocateDenseBlock();

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultWCeTask> tasks = new ArrayList<MatrixMultWCeTask>();
        int blklen = (int) (Math.ceil((double) mW.rlen / k));
        for (int i = 0; i < k & i * blklen < mW.rlen; i++)
            tasks.add(new MatrixMultWCeTask(mW, mU, mV, eps, wt, i * blklen,
                    Math.min((i + 1) * blklen, mW.rlen)));
        List<Future<Double>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        //aggregate partial results
        sumScalarResults(taskret, ret);
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }

    //System.out.println("MMWCe "+wt.toString()+" k="+k+" ("+mW.isInSparseFormat()+","+mW.getNumRows()+","+mW.getNumColumns()+","+mW.getNonZeros()+")x" +
    //                 "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop());
}

From source file:org.apache.sysml.runtime.matrix.data.LibMatrixMult.java

public static void matrixMultWSLoss(MatrixBlock mX, MatrixBlock mU, MatrixBlock mV, MatrixBlock mW,
        MatrixBlock ret, WeightsType wt, int k) throws DMLRuntimeException {
    //check for empty result
    if (wt == WeightsType.POST && mW.isEmptyBlock(false)
            || wt == WeightsType.POST_NZ && mX.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;/*from   www .ja va  2 s .c  om*/
    }

    //check no parallelization benefit (fallback to sequential)
    //no need to check isThreadSafe (scalar output)
    if (mX.rlen == 1) {
        matrixMultWSLoss(mX, mU, mV, mW, ret, wt);
        return;
    }

    //Timing time = new Timing(true);

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultWSLossTask> tasks = new ArrayList<MatrixMultWSLossTask>();
        int blklen = (int) (Math.ceil((double) mX.rlen / k));
        for (int i = 0; i < k & i * blklen < mX.rlen; i++)
            tasks.add(new MatrixMultWSLossTask(mX, mU, mV, mW, wt, i * blklen,
                    Math.min((i + 1) * blklen, mX.rlen)));
        List<Future<Double>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        //aggregate partial results
        sumScalarResults(taskret, ret);
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }

    //System.out.println("MMWSLoss "+wt.toString()+" k="+k+" ("+mX.isInSparseFormat()+","+mX.getNumRows()+","+mX.getNumColumns()+","+mX.getNonZeros()+")x" +
    //                   "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop());
}

From source file:org.apache.sysml.runtime.matrix.data.LibMatrixMult.java

public static void matrixMultWuMM(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, MatrixBlock ret, WUMMType wt,
        ValueFunction fn, int k) throws DMLRuntimeException {
    //check for empty result
    if (mW.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;//ww w  .  ja  v a  2 s  .  c  om
    }

    //check no parallelization benefit (fallback to sequential)
    if (mW.rlen == 1 || !MatrixBlock.isThreadSafe(mW.sparse)) {
        matrixMultWuMM(mW, mU, mV, ret, wt, fn);
        return;
    }

    //Timing time = new Timing(true);

    //pre-processing
    ret.sparse = mW.sparse;
    ret.allocateDenseOrSparseBlock();

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultWuTask> tasks = new ArrayList<MatrixMultWuTask>();
        int blklen = (int) (Math.ceil((double) mW.rlen / k));
        for (int i = 0; i < k & i * blklen < mW.rlen; i++)
            tasks.add(new MatrixMultWuTask(mW, mU, mV, ret, wt, fn, i * blklen,
                    Math.min((i + 1) * blklen, mW.rlen)));
        //execute tasks
        List<Future<Long>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        //aggregate partial nnz and check for errors
        ret.nonZeros = 0; //reset after execute
        for (Future<Long> task : taskret)
            ret.nonZeros += task.get();
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }

    //post-processing (nnz maintained in parallel)
    ret.examSparsity();

    //System.out.println("MMWu "+wt.toString()+" k="+k+" ("+mW.isInSparseFormat()+","+mW.getNumRows()+","+mW.getNumColumns()+","+mW.getNonZeros()+")x" +
    //                   "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop() + ".");
}

From source file:com.ibm.bi.dml.runtime.matrix.data.LibMatrixMult.java

/**
 * Performs a parallel matrix multiplication chain operation of type t(X)%*%(X%*%v) or t(X)%*%(w*(X%*%v)).
 * The parameter k (k>=1) determines the max parallelism k' with k'=min(k, vcores, m1.rlen).
 * /*  w ww.  ja v  a  2 s  .  c o m*/
 * NOTE: This multi-threaded mmchain operation has additional memory requirements of k*ncol(X)*8bytes 
 * for partial aggregation. Current max memory: 256KB; otherwise redirectly to sequential execution.
 * 
 * @param mX
 * @param mV
 * @param mW
 * @param ret
 * @param ct
 * @param k
 * @throws DMLRuntimeException
 * @throws DMLUnsupportedOperationException
 */
public static void matrixMultChain(MatrixBlock mX, MatrixBlock mV, MatrixBlock mW, MatrixBlock ret,
        ChainType ct, int k) throws DMLRuntimeException, DMLUnsupportedOperationException {
    //check inputs / outputs (after that mV and mW guaranteed to be dense)
    if (mX.isEmptyBlock(false) || mV.isEmptyBlock(false) || (mW != null && mW.isEmptyBlock(false))) {
        ret.examSparsity(); //turn empty dense into sparse
        return;
    }

    //check too high additional memory requirements (fallback to sequential)
    //check too small workload in terms of flops (fallback to sequential too)
    if (8L * mV.rlen * k > MEM_OVERHEAD_THRESHOLD || 4L * mX.rlen * mX.clen < PAR_MINFLOP_THRESHOLD) {
        matrixMultChain(mX, mV, mW, ret, ct);
        return;
    }

    //Timing time = new Timing(true);

    //pre-processing
    ret.sparse = false;
    ret.allocateDenseBlock();

    //core matrix mult chain computation
    //(currently: always parallelization over number of rows)
    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultChainTask> tasks = new ArrayList<MatrixMultChainTask>();
        int blklen = (int) (Math.ceil((double) mX.rlen / k));
        blklen += (blklen % 24 != 0) ? 24 - blklen % 24 : 0;
        for (int i = 0; i < k & i * blklen < mX.rlen; i++)
            tasks.add(new MatrixMultChainTask(mX, mV, mW, ret, ct, i * blklen,
                    Math.min((i + 1) * blklen, mX.rlen)));
        pool.invokeAll(tasks);
        pool.shutdown();
        //aggregate partial results
        for (MatrixMultChainTask task : tasks)
            vectAdd(task.getResult().denseBlock, ret.denseBlock, 0, 0, mX.clen);
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }

    //post-processing
    ret.recomputeNonZeros();
    ret.examSparsity();

    //System.out.println("MMChain "+ct.toString()+" k="+k+" ("+mX.isInSparseFormat()+","+mX.getNumRows()+","+mX.getNumColumns()+","+mX.getNonZeros()+")x" +
    //                    "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop());
}

From source file:org.apache.sysml.runtime.matrix.data.LibMatrixMult.java

public static void matrixMultWSigmoid(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, MatrixBlock ret,
        WSigmoidType wt, int k) throws DMLRuntimeException {
    //check for empty result
    if (mW.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;//www .ja v a  2 s  .  com
    }

    //check no parallelization benefit (fallback to sequential)
    if (mW.rlen == 1 || !MatrixBlock.isThreadSafe(mW.sparse)) {
        matrixMultWSigmoid(mW, mU, mV, ret, wt);
        return;
    }

    //Timing time = new Timing(true);

    //pre-processing
    ret.sparse = mW.sparse;
    ret.allocateDenseOrSparseBlock();

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultWSigmoidTask> tasks = new ArrayList<MatrixMultWSigmoidTask>();
        int blklen = (int) (Math.ceil((double) mW.rlen / k));
        for (int i = 0; i < k & i * blklen < mW.rlen; i++)
            tasks.add(new MatrixMultWSigmoidTask(mW, mU, mV, ret, wt, i * blklen,
                    Math.min((i + 1) * blklen, mW.rlen)));
        //execute tasks
        List<Future<Long>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        //aggregate partial nnz and check for errors
        ret.nonZeros = 0; //reset after execute
        for (Future<Long> task : taskret)
            ret.nonZeros += task.get();
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }

    //post-processing (nnz maintained in parallel)
    ret.examSparsity();

    //System.out.println("MMWSig "+wt.toString()+" k="+k+" ("+mW.isInSparseFormat()+","+mW.getNumRows()+","+mW.getNumColumns()+","+mW.getNonZeros()+")x" +
    //                   "("+mV.isInSparseFormat()+","+mV.getNumRows()+","+mV.getNumColumns()+","+mV.getNonZeros()+") in "+time.stop() + ".");
}

From source file:com.ibm.bi.dml.runtime.matrix.data.LibMatrixMult.java

/**
 * //from w w  w .  j  a  va2  s . c o  m
 * @param m1
 * @param ret
 * @param leftTranspose
 * @param k
 * @throws DMLUnsupportedOperationException
 * @throws DMLRuntimeException
 */
public static void matrixMultTransposeSelf(MatrixBlock m1, MatrixBlock ret, boolean leftTranspose, int k)
        throws DMLUnsupportedOperationException, DMLRuntimeException {
    //check inputs / outputs
    if (m1.isEmptyBlock(false)) {
        ret.examSparsity(); //turn empty dense into sparse
        return;
    }

    //check no parallelization benefit (fallback to sequential)
    //check too small workload in terms of flops (fallback to sequential too)
    if (ret.rlen == 1 || leftTranspose && 1L * m1.rlen * m1.clen * m1.clen < PAR_MINFLOP_THRESHOLD
            || !leftTranspose && 1L * m1.clen * m1.rlen * m1.rlen < PAR_MINFLOP_THRESHOLD) {
        matrixMultTransposeSelf(m1, ret, leftTranspose);
        return;
    }

    //Timing time = new Timing(true);

    //pre-processing
    m1 = prepMatrixMultTransposeSelfInput(m1, leftTranspose);
    ret.sparse = false;
    ret.allocateDenseBlock();

    //core multi-threaded matrix mult computation
    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<MatrixMultTransposeTask> tasks = new ArrayList<MatrixMultTransposeTask>();
        //load balance via #tasks=2k due to triangular shape 
        int blklen = (int) (Math.ceil((double) ret.rlen / (2 * k)));
        for (int i = 0; i < 2 * k & i * blklen < ret.rlen; i++)
            tasks.add(new MatrixMultTransposeTask(m1, ret, leftTranspose, i * blklen,
                    Math.min((i + 1) * blklen, ret.rlen)));
        pool.invokeAll(tasks);
        pool.shutdown();
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }

    //post-processing
    copyUpperToLowerTriangle(ret);
    ret.recomputeNonZeros();
    ret.examSparsity();

    //System.out.println("TSMM k="+k+" ("+m1.isInSparseFormat()+","+m1.getNumRows()+","+m1.getNumColumns()+","+m1.getNonZeros()+","+leftTranspose+") in "+time.stop());
}