Example usage for org.apache.hadoop.mapreduce Job submit

List of usage examples for org.apache.hadoop.mapreduce Job submit

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. submit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job submit.

Prototype

public void submit() throws IOException, InterruptedException, ClassNotFoundException

Source Link

Document

Submit the job to the cluster and return immediately.

Usage

From source file:org.pentaho.hadoop.shim.emr32.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {// w ww  . ja  v a  2s  .  c  om
        Job job = ((org.pentaho.hadoop.shim.emr32.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

From source file:org.pentaho.hadoop.shim.emr34.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {/* w w  w . ja  v  a2 s.c  o m*/
        Job job = ((org.pentaho.hadoop.shim.emr34.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

From source file:org.pentaho.hadoop.shim.hdp21.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {/*from   ww  w . ja v  a  2s .c  o  m*/
        Job job = ((org.pentaho.hadoop.shim.hdp21.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

From source file:org.pentaho.hadoop.shim.hdp22.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {/* w ww . ja va2  s .  c o m*/
        Job job = ((org.pentaho.hadoop.shim.hdp22.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

From source file:org.pentaho.hadoop.shim.mapr401.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {// w ww.j  a  v a 2s.c  o  m
        Job job = ((org.pentaho.hadoop.shim.mapr401.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

From source file:org.pentaho.hadoop.shim.mapr410.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {//from   ww  w .j  ava 2  s  .c o m
        Job job = ((org.pentaho.hadoop.shim.mapr410.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

From source file:org.qcri.pca.CompositeJob.java

/**
 * Computes XtX and YtX//from   w  w  w . ja va2 s .c o  m
 * 
 * Xc = (Y - Ym) * MEM = Y * MEM - Ym * MEM = X - Xm
 * 
 * XtX = (X - Xm)' * (X - Xm) YtX = (Y - Ym)' * (Y - Ym)
 * 
 * @param conf
 *          the configuration
 * @param matrixInputPath
 *          Y
 * @param inMemMatrixDir
 *          MEM, where X = Y * MEM
 * @param inMemMatrixNumRows
 *          MEM.rows
 * @param inMemMatrixNumCols
 *          MEM.cols
 * @param ymPath
 *          Ym
 * @param xmPath
 *          Xm
 * @param matrixOutputPath
 *          YtX
 * @throws IOException
 * @throws InterruptedException
 * @throws ClassNotFoundException
 */
public void run(Configuration conf, Path matrixInputPath, String inMemMatrixDir, int inMemMatrixNumRows,
        int inMemMatrixNumCols, String ymPath, String xmPath, Path matrixOutputPath)
        throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(MATRIXINMEMORY, inMemMatrixDir);
    conf.setInt(MATRIXINMEMORYROWS, inMemMatrixNumRows);
    conf.setInt(MATRIXINMEMORYCOLS, inMemMatrixNumCols);
    conf.set(YMPATH, ymPath);
    conf.set(XMPATH, xmPath);
    Path xtxOutputPath = getXtXPathBasedOnYm(new Path(ymPath));
    conf.set(XTXPATH, xtxOutputPath.toString());
    Job job = new Job(conf);
    job.setJobName("CompositeJob-" + matrixInputPath.getName());
    job.setJarByClass(CompositeJob.class);
    FileSystem fs = FileSystem.get(matrixInputPath.toUri(), conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    FileInputFormat.addInputPath(job, matrixInputPath);
    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileOutputFormat.setOutputPath(job, matrixOutputPath);
    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setMapOutputKeyClass(CompositeWritable.class);
    job.setMapOutputValueClass(VectorWritable.class);
    job.setSortComparatorClass(CompositeWritable.class);
    job.setGroupingComparatorClass(CompositeWritable.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(VectorWritable.class);
    job.submit();
    job.waitForCompletion(true);
}

From source file:org.qcri.pca.MeanAndSpanJob.java

/**
 * Job for calculating column-wise mean and span of a
 * {@link DistributedRowMatrix}//from  w  w w .j ava 2 s  .c o m
 * 
 * @param initialConf
 *          the initial configuration
 * @param inputPath
 *          the path to the matrix
 * @param outputVectorTmpPath
 *          the path to which the result vectors will be written
 * @throws IOException
 *           in case of any error
 */
public void run(Configuration initialConf, Path inputPath, Path outputVectorTmpPath) throws IOException {
    try {
        Job job = new Job(initialConf);
        job.setJobName("MeanAndSpan");
        job.setJarByClass(MeanAndSpanJob.class);
        FileOutputFormat.setOutputPath(job, outputVectorTmpPath);
        FileInputFormat.addInputPath(job, inputPath);
        job.setInputFormatClass(SequenceFileInputFormat.class);
        job.setOutputFormatClass(SequenceFileOutputFormat.class);

        job.setMapperClass(MeanAndSpanMapper.class);
        job.setReducerClass(MeanAndSpanReducer.class);
        job.setNumReduceTasks(1);// it has to be one
        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(VectorWritable.class);
        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(VectorWritable.class);
        job.submit();
        job.waitForCompletion(true);
    } catch (Throwable thr) {
        thr.printStackTrace();
        if (thr instanceof IOException)
            throw (IOException) thr;
        else
            throw new IOException(thr);
    }
}

From source file:org.qcri.pca.Norm2Job.java

public void run(Configuration conf, Path matrixInputPath, String meanSpanFileName, Path matrixOutputPath)
        throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(MEANSPANOPTION, meanSpanFileName);
    Job job = new Job(conf);
    job.setJobName("Norm2Job");
    job.setJarByClass(Norm2Job.class);
    FileSystem fs = FileSystem.get(matrixInputPath.toUri(), conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    FileInputFormat.addInputPath(job, matrixInputPath);
    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    FileOutputFormat.setOutputPath(job, matrixOutputPath);
    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);
    job.setNumReduceTasks(1);/*  ww  w.  ja  va2  s. c  o  m*/
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(DoubleWritable.class);
    job.submit();
    job.waitForCompletion(true);
}

From source file:org.qcri.pca.NormalizeJob.java

public void run(Configuration conf, Path matrixInputPath, String meanSpanFileName, Path matrixOutputPath,
        double sampleRate) throws IOException, InterruptedException, ClassNotFoundException {
    conf.set(MEANSPANOPTION, meanSpanFileName);
    conf.setFloat(SAMPLERATE, (float) sampleRate);
    Job job = new Job(conf);
    job.setJobName("Normalize");
    job.setJarByClass(NormalizeJob.class);
    FileSystem fs = FileSystem.get(matrixInputPath.toUri(), conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    FileInputFormat.addInputPath(job, matrixInputPath);
    job.setInputFormatClass(SequenceFileInputFormat.class);
    FileOutputFormat.setOutputPath(job, matrixOutputPath);
    job.setMapperClass(NormalizeMapper.class);
    job.setNumReduceTasks(0);//from  ww w .  j ava  2  s  .  co m
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(VectorWritable.class);
    job.submit();
    job.waitForCompletion(true);
}