Example usage for org.apache.hadoop.mapreduce Job submit

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job submit.

Prototype

public void submit() throws IOException, InterruptedException, ClassNotFoundException 

Source Link

Document

Submit the job to the cluster and return immediately.

Usage

From source file:org.mrgeo.mapreduce.MapReduceUtils.java

License:Apache License

public static boolean runJob(Job job, Progress progress, JobListener jl)
        throws JobFailedException, JobCancelledException {
    boolean success = false;
    if (jl != null) {
        //append job id to the job name for easy identification
        job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName());
        jl.addJob(job);/*from  www  .  ja  v  a 2s.c o m*/
    }

    long start = System.currentTimeMillis();
    log.info("Running job {}", job.getJobName());

    try {
        job.submit();
        log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start));
        if (progress == null) {
            job.waitForCompletion(true);
        } else {
            float initP = progress.get();
            float percentP = 100 - initP;
            while (job.isComplete() == false) {
                float p = job.mapProgress() * .9f + job.reduceProgress() * .1f;
                progress.set(p * percentP + initP);
                try {
                    Thread.sleep(500);
                } catch (InterruptedException e) {
                    log.info("Job Cancelled by user");
                    throw new JobCancelledException("Job Cancelled by user.");
                }
            }
        }

        log.info("Job {} time: {}ms", job.getJobName(), (System.currentTimeMillis() - start));

        if (job.isSuccessful() == false) {
            throw new JobFailedException("Job failed: " + job.getTrackingURL());
        }
        success = job.isSuccessful();
    } catch (InterruptedException e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    }
    // when submitting jobs under JBoss, Exception doesn't appear to be caught
    catch (Throwable e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    }
    return success;
}

---

From source file:org.mrgeo.mapreduce.MapReduceUtils.java

License:Apache License

public static void runJobAsynchronously(Job job, JobListener jl)
        throws IOException, JobFailedException, JobCancelledException {
    if (jl != null) {
        //append job id to the job name for easy identification
        job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName());
        jl.addJob(job);// w ww  .j  a  v a  2 s. co  m
    }

    try {
        long start = System.currentTimeMillis();
        log.info("Running asynchronous job {}", job.getJobName());
        job.submit();
        log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start));
    } catch (InterruptedException e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
        throw new JobFailedException(e.getMessage());
    }
}

---

From source file:org.mrgeo.vector.mrsvector.OSMTileIngester.java

License:Apache License

private boolean buildTiles() {
    try {// w  w  w.j a  v a 2  s .co m
        final Job job = new Job(config);
        HadoopUtils.setJar(job, this.getClass());

        final String now = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss").format(new Date());

        final String jobName = "BuildTiles_" + now + "_" + UUID.randomUUID().toString();
        job.setJobName(jobName);

        final Configuration conf = job.getConfiguration();

        conf.setInt(ZOOMLEVEL, zoomlevel);
        final int tilesize = Integer.parseInt(MrGeoProperties.getInstance()
                .getProperty(MrGeoConstants.MRGEO_MRS_TILESIZE, MrGeoConstants.MRGEO_MRS_TILESIZE_DEFAULT));
        conf.setInt(TILESIZE, tilesize);
        conf.set(OUTPUT, tmpDir.toString());

        conf.setInt(GRANULATIRY, granularity);
        conf.setLong(LATOFFSET, latOffset);
        conf.setLong(LONOFFSET, lonOffset);

        job.setInputFormatClass(SequenceFileInputFormat.class);

        final Path tilesPath = new Path(tmpDir, TILEIDS + "/*/part*");
        HadoopVectorUtils.addInputPath(job, tilesPath);

        job.setReducerClass(ProcessTilesReducer.class);

        final Path output = new Path(tmpDir, VECTORTILES);

        HadoopFileUtils.delete(output);

        MrsImageOutputFormatProvider ofProvider = MrsImageDataProvider.setupMrsPyramidOutputFormat(job,
                output.toString(), datasetBounds, zoomlevel, tilesize, protectionLevel, providerProperties);
        //FileOutputFormat.setOutputPath(job, outputWithZoom);

        job.setMapOutputKeyClass(TileIdWritable.class);
        job.setMapOutputValueClass(Text.class);

        job.setOutputKeyClass(TileIdWritable.class);
        job.setOutputValueClass(VectorTileWritable.class);

        try {
            job.submit();
            final boolean success = job.waitForCompletion(true);

            if (success) {
                ofProvider.teardown(job);
                MrsVectorPyramid.calculateMetadata(output.toString(), zoomlevel, tilesize, datasetBounds,
                        protectionLevel);
                return true;
            }

        } catch (final InterruptedException e) {
            e.printStackTrace();
        } catch (final ClassNotFoundException e) {
            e.printStackTrace();
        }
    } catch (final IOException e) {
        e.printStackTrace();
    }
    return false;

}

---

From source file:org.mrgeo.vector.mrsvector.OSMTileIngester.java

License:Apache License

private boolean processNodes() {
    try {//from  w  w w  . j a v a2 s . c om
        final Job job = new Job(config);
        HadoopUtils.setJar(job, this.getClass());

        final String now = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss").format(new Date());

        final String jobName = "ProcesNodes_" + now + "_" + UUID.randomUUID().toString();
        job.setJobName(jobName);

        final Configuration conf = job.getConfiguration();

        conf.setInt(ZOOMLEVEL, zoomlevel);
        conf.setInt(TILESIZE, Integer.parseInt(MrGeoProperties.getInstance()
                .getProperty(MrGeoConstants.MRGEO_MRS_TILESIZE, MrGeoConstants.MRGEO_MRS_TILESIZE_DEFAULT)));
        conf.set(OUTPUT, tmpDir.toString());

        conf.setInt(GRANULATIRY, granularity);
        conf.setLong(LATOFFSET, latOffset);
        conf.setLong(LONOFFSET, lonOffset);

        job.setInputFormatClass(SequenceFileInputFormat.class);

        final Path nodesPath = new Path(tmpDir, NODES);
        HadoopVectorUtils.addInputPath(job, nodesPath);

        job.setReducerClass(ProcessNodesReducer.class);

        job.setOutputFormatClass(SequenceFileOutputFormat.class);

        final Path output = new Path(tmpDir, TILEIDS + "/" + NODES);
        HadoopFileUtils.delete(output);
        FileOutputFormat.setOutputPath(job, output);

        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(Text.class);

        job.setOutputKeyClass(TileIdWritable.class);
        job.setOutputValueClass(Text.class);

        try {
            job.submit();
            final boolean success = job.waitForCompletion(true);

            if (success) {
                return true;
            }

        } catch (final InterruptedException e) {
            e.printStackTrace();
        } catch (final ClassNotFoundException e) {
            e.printStackTrace();
        }
    } catch (final IOException e) {
        e.printStackTrace();
    }
    return false;
}

---

From source file:org.mrgeo.vector.mrsvector.OSMTileIngester.java

License:Apache License

private boolean processRelations() {
    try {//from  w  w w  . j  a  v a  2s  . c om
        int runCnt = 1;

        while (true) {
            final Job job = new Job(config);
            HadoopUtils.setJar(job, this.getClass());

            final String now = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss").format(new Date());

            final String jobName = "ProcesRelations_" + runCnt + "_" + now + "_" + UUID.randomUUID().toString();
            job.setJobName(jobName);

            final Configuration conf = job.getConfiguration();

            conf.setInt(ZOOMLEVEL, zoomlevel);
            conf.setInt(TILESIZE, Integer.parseInt(MrGeoProperties.getInstance().getProperty(
                    MrGeoConstants.MRGEO_MRS_TILESIZE, MrGeoConstants.MRGEO_MRS_TILESIZE_DEFAULT)));
            conf.set(OUTPUT, tmpDir.toString());

            conf.setInt(GRANULATIRY, granularity);
            conf.setLong(LATOFFSET, latOffset);
            conf.setLong(LONOFFSET, lonOffset);
            conf.setInt(RELATION_RUN, runCnt);

            job.setInputFormatClass(SequenceFileInputFormat.class);

            final Path relationsPath;
            if (runCnt <= 1) {
                relationsPath = new Path(tmpDir, RELATIONS);
            } else {
                relationsPath = new Path(tmpDir, RELATIONS + "_" + (runCnt - 1));
            }
            HadoopVectorUtils.addInputPath(job, relationsPath);

            job.setOutputFormatClass(SequenceFileOutputFormat.class);

            final Path output = new Path(tmpDir, TILEIDS + "/" + RELATIONS + "_" + runCnt);
            HadoopFileUtils.delete(output);
            FileOutputFormat.setOutputPath(job, output);

            job.setReducerClass(ProcessRelationsReducer.class);

            job.setMapOutputKeyClass(LongWritable.class);
            job.setMapOutputValueClass(Text.class);

            job.setOutputKeyClass(TileIdWritable.class);
            job.setOutputValueClass(Text.class);

            boolean success = false;
            try {
                job.submit();
                success = job.waitForCompletion(true);
            } catch (final InterruptedException e) {
                e.printStackTrace();
            } catch (final ClassNotFoundException e) {
                e.printStackTrace();
            }

            if (success) {

                final Path rp = new Path(tmpDir, RELATIONS + "_" + runCnt);

                // did we make a relations file?
                if (!HadoopFileUtils.exists(rp)) {
                    return true;
                }
            }
            runCnt++;

            if (runCnt > 5) {
                return true;
            }
        }
    } catch (final IOException e) {
        e.printStackTrace();
    }
    return false;
}

---

From source file:org.mrgeo.vector.mrsvector.OSMTileIngester.java

License:Apache License

private boolean processWays() {
    try {/*from   ww w.  ja v a  2s  .  co m*/
        final Job job = new Job(config);
        HadoopUtils.setJar(job, this.getClass());

        final String now = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss").format(new Date());

        final String jobName = "ProcesWays_" + now + "_" + UUID.randomUUID().toString();
        job.setJobName(jobName);

        final Configuration conf = job.getConfiguration();

        conf.setInt(ZOOMLEVEL, zoomlevel);
        conf.setInt(TILESIZE, Integer.parseInt(MrGeoProperties.getInstance()
                .getProperty(MrGeoConstants.MRGEO_MRS_TILESIZE, MrGeoConstants.MRGEO_MRS_TILESIZE_DEFAULT)));
        conf.set(OUTPUT, tmpDir.toString());

        conf.setInt(GRANULATIRY, granularity);
        conf.setLong(LATOFFSET, latOffset);
        conf.setLong(LONOFFSET, lonOffset);

        job.setInputFormatClass(SequenceFileInputFormat.class);

        final Path waysPath = new Path(tmpDir, WAYS);
        HadoopVectorUtils.addInputPath(job, waysPath);

        job.setReducerClass(ProcessWaysReducer.class);

        job.setOutputFormatClass(SequenceFileOutputFormat.class);

        final Path output = new Path(tmpDir, TILEIDS + "/" + WAYS);
        HadoopFileUtils.delete(output);
        FileOutputFormat.setOutputPath(job, output);

        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(Text.class);

        job.setOutputKeyClass(TileIdWritable.class);
        job.setOutputValueClass(Text.class);

        try {
            job.submit();
            final boolean success = job.waitForCompletion(true);

            if (success) {
                return true;
            }
        } catch (final InterruptedException e) {
            e.printStackTrace();
        } catch (final ClassNotFoundException e) {
            e.printStackTrace();
        }
    } catch (final IOException e) {
        e.printStackTrace();
    }
    return false;
}

---

From source file:org.pentaho.hadoop.shim.cdh51.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {//from   w w w. j  a v  a2s. co m
        Job job = ((org.pentaho.hadoop.shim.cdh51.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

---

From source file:org.pentaho.hadoop.shim.cdh52.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {//w w w .ja v  a 2s  .c  o m
        Job job = ((org.pentaho.hadoop.shim.cdh52.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

---

From source file:org.pentaho.hadoop.shim.cdh53.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {/*from  w w w  .j  a  va 2  s .  com*/
        Job job = ((org.pentaho.hadoop.shim.cdh53.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

---

From source file:org.pentaho.hadoop.shim.cdh54.HadoopShim.java

License:Apache License

@Override
public RunningJob submitJob(org.pentaho.hadoop.shim.api.Configuration c) throws IOException {
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
    try {//from   www.ja  v a 2 s  .  co  m
        Job job = ((org.pentaho.hadoop.shim.cdh54.ConfigurationProxyV2) c).getJob();
        job.submit();
        return new RunningJobProxyV2(job);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    } finally {
        Thread.currentThread().setContextClassLoader(cl);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• »