Example usage for org.apache.spark SparkJobInfo stageIds

Introduction

In this page you can find the example usage for org.apache.spark SparkJobInfo stageIds.

Prototype

int[] stageIds();

Source Link

Usage

From source file:cn.com.bsfit.frms.spark.StatusTrackerDemo.java

License:Apache License

public static void main(String[] args) throws Exception {
    SparkSession spark = SparkSession.builder().appName(APP_NAME).getOrCreate();

    final JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());

    // Example of implementing a progress reporter for a simple job.
    JavaRDD<Integer> rdd = jsc.parallelize(Arrays.asList(1, 2, 3, 4, 5), 5)
            .map(new IdentityWithDelay<Integer>());
    JavaFutureAction<List<Integer>> jobFuture = rdd.collectAsync();
    while (!jobFuture.isDone()) {
        Thread.sleep(1000); // 1 second
        List<Integer> jobIds = jobFuture.jobIds();
        if (jobIds.isEmpty()) {
            continue;
        }/*ww w . j  a va2  s.co m*/
        int currentJobId = jobIds.get(jobIds.size() - 1);
        SparkJobInfo jobInfo = jsc.statusTracker().getJobInfo(currentJobId);
        SparkStageInfo stageInfo = jsc.statusTracker().getStageInfo(jobInfo.stageIds()[0]);
        System.out.println(stageInfo.numTasks() + " tasks total: " + stageInfo.numActiveTasks() + " active, "
                + stageInfo.numCompletedTasks() + " complete");
    }

    System.out.println("Job results are: " + jobFuture.get());
    jsc.close();
    spark.stop();
}

---

From source file:com.andado.spark.examples.JavaStatusTrackerDemo.java

License:Apache License

public static void main(String[] args) throws Exception {
    SparkSession spark = SparkSession.builder().appName(APP_NAME).getOrCreate();

    final JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());

    // Example of implementing a progress reporter for a simple job.
    JavaRDD<Integer> rdd = jsc.parallelize(Arrays.asList(1, 2, 3, 4, 5), 5)
            .map(new IdentityWithDelay<Integer>());
    JavaFutureAction<List<Integer>> jobFuture = rdd.collectAsync();
    while (!jobFuture.isDone()) {
        Thread.sleep(1000); // 1 second
        List<Integer> jobIds = jobFuture.jobIds();
        if (jobIds.isEmpty()) {
            continue;
        }/*from   w  ww. ja  v a2  s.c  om*/
        int currentJobId = jobIds.get(jobIds.size() - 1);
        SparkJobInfo jobInfo = jsc.statusTracker().getJobInfo(currentJobId);
        SparkStageInfo stageInfo = jsc.statusTracker().getStageInfo(jobInfo.stageIds()[0]);
        System.out.println(stageInfo.numTasks() + " tasks total: " + stageInfo.numActiveTasks() + " active, "
                + stageInfo.numCompletedTasks() + " complete");
    }

    System.out.println("Job results are: " + jobFuture.get());
    spark.stop();
}

---

From source file:com.dmall.order.realtime.tacking.function.JavaStatusTrackerDemo.java

License:Apache License

public static void main(String[] args) throws Exception {
    SparkSession spark = SparkSession.builder().appName(APP_NAME).master("spark://192.168.184.128:7077")
            .getOrCreate();/*from   ww  w.  j a va 2s . c om*/

    final JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());
    jsc.addJar("D:\\learn\\java\\learn-spark\\target\\spark.jar");
    // Example of implementing a progress reporter for a simple job.
    JavaRDD<Integer> rdd = jsc.parallelize(Arrays.asList(1, 2, 3, 4, 5), 5)
            .map(new IdentityWithDelay<Integer>());
    JavaFutureAction<List<Integer>> jobFuture = rdd.collectAsync();
    while (!jobFuture.isDone()) {
        Thread.sleep(1000); // 1 second
        List<Integer> jobIds = jobFuture.jobIds();
        if (jobIds.isEmpty()) {
            continue;
        }
        int currentJobId = jobIds.get(jobIds.size() - 1);
        SparkJobInfo jobInfo = jsc.statusTracker().getJobInfo(currentJobId);
        SparkStageInfo stageInfo = jsc.statusTracker().getStageInfo(jobInfo.stageIds()[0]);
        System.out.println(stageInfo.numTasks() + " tasks total: " + stageInfo.numActiveTasks() + " active, "
                + stageInfo.numCompletedTasks() + " complete");
    }

    System.out.println("Job results are: " + jobFuture.get());
    spark.stop();
}

---

From source file:com.hxr.bigdata.spark.example141.JavaStatusTrackerDemo.java

License:Apache License

public static void main(String[] args) throws Exception {
    SparkConf sparkConf = new SparkConf().setAppName(APP_NAME);
    final JavaSparkContext sc = new JavaSparkContext(sparkConf);

    // Example of implementing a progress reporter for a simple job.
    JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5), 5)
            .map(new IdentityWithDelay<Integer>());
    JavaFutureAction<List<Integer>> jobFuture = rdd.collectAsync();
    while (!jobFuture.isDone()) {
        Thread.sleep(1000); // 1 second
        List<Integer> jobIds = jobFuture.jobIds();
        if (jobIds.isEmpty()) {
            continue;
        }//w  w  w  .j  a v a2s  .  co  m
        int currentJobId = jobIds.get(jobIds.size() - 1);
        SparkJobInfo jobInfo = sc.statusTracker().getJobInfo(currentJobId);
        SparkStageInfo stageInfo = sc.statusTracker().getStageInfo(jobInfo.stageIds()[0]);
        System.out.println(stageInfo.numTasks() + " tasks total: " + stageInfo.numActiveTasks() + " active, "
                + stageInfo.numCompletedTasks() + " complete");
    }

    System.out.println("Job results are: " + jobFuture.get());
    sc.stop();
}

---

From source file:gtl.spark.java.example.apache.JavaStatusTrackerDemo.java

License:Apache License

public static void main(String[] args) throws Exception {
    SparkSession spark = SparkSession.builder().appName(APP_NAME).getOrCreate();

    JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());

    // Example of implementing a progress reporter for a simple job.
    JavaRDD<Integer> rdd = jsc.parallelize(Arrays.asList(1, 2, 3, 4, 5), 5).map(new IdentityWithDelay<>());
    JavaFutureAction<List<Integer>> jobFuture = rdd.collectAsync();
    while (!jobFuture.isDone()) {
        Thread.sleep(1000); // 1 second
        List<Integer> jobIds = jobFuture.jobIds();
        if (jobIds.isEmpty()) {
            continue;
        }/*from  ww  w.  j  a v  a2  s.c  o m*/
        int currentJobId = jobIds.get(jobIds.size() - 1);
        SparkJobInfo jobInfo = jsc.statusTracker().getJobInfo(currentJobId);
        SparkStageInfo stageInfo = jsc.statusTracker().getStageInfo(jobInfo.stageIds()[0]);
        System.out.println(stageInfo.numTasks() + " tasks total: " + stageInfo.numActiveTasks() + " active, "
                + stageInfo.numCompletedTasks() + " complete");
    }

    System.out.println("Job results are: " + jobFuture.get());
    spark.stop();
}

---

From source file:org.apache.hadoop.hive.ql.exec.spark.status.impl.LocalSparkJobStatus.java

License:Apache License

@Override
public int[] getStageIds() {
    SparkJobInfo sparkJobInfo = getJobInfo();
    return sparkJobInfo == null ? new int[0] : sparkJobInfo.stageIds();
}

---

From source file:org.apache.hadoop.hive.ql.exec.spark.status.impl.RemoteSparkJobStatus.java

License:Apache License

@Override
public int[] getStageIds() throws HiveException {
    SparkJobInfo sparkJobInfo = getSparkJobInfo();
    return sparkJobInfo != null ? sparkJobInfo.stageIds() : new int[0];
}

---