Logistic regression based classification using Machine Learning Library via apache spark - Java Big Data

Java examples for Big Data:apache spark

  1. HOME
  2. Java
  3. Big Data
  4. apache spark

Description

Logistic regression based classification using Machine Learning Library via apache spark

Demo Code

/*//w  w w .ja  va  2s.c  o m
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eduonix.spark.mllib;

import java.util.regex.Pattern;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;

import org.apache.spark.mllib.classification.LogisticRegressionWithSGD;
import org.apache.spark.mllib.classification.LogisticRegressionModel;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;

/**
 * Logistic regression based classification using ML Lib.
 */
public final class JavaLR {

    static class ParsePoint implements Function<String, LabeledPoint> {
        private static final Pattern COMMA = Pattern.compile(",");
        private static final Pattern SPACE = Pattern.compile(" ");

        @Override
        public LabeledPoint call(String line) {
            String[] parts = COMMA.split(line);
            double y = Double.parseDouble(parts[0]);
            String[] tok = SPACE.split(parts[1]);
            double[] x = new double[tok.length];
            for (int i = 0; i < tok.length; ++i) {
                x[i] = Double.parseDouble(tok[i]);
            }
            return new LabeledPoint(y, Vectors.dense(x));
        }
    }

    public static void main(String[] args) {
        if (args.length != 3) {
            System.err
                    .println("Usage: JavaLR <input_dir> <step_size> <niters>");
            System.exit(1);
        }
        SparkConf sparkConf = new SparkConf().setAppName("JavaLR");
        JavaSparkContext sc = new JavaSparkContext(sparkConf);
        JavaRDD<String> lines = sc.textFile(args[0]);
        JavaRDD<LabeledPoint> points = lines.map(new ParsePoint()).cache();
        double stepSize = Double.parseDouble(args[1]);
        int iterations = Integer.parseInt(args[2]);

        // Another way to configure LogisticRegression
        //
        // LogisticRegressionWithSGD lr = new LogisticRegressionWithSGD();
        // lr.optimizer().setNumIterations(iterations)
        //               .setStepSize(stepSize)
        //               .setMiniBatchFraction(1.0);
        // lr.setIntercept(true);
        // LogisticRegressionModel model = lr.train(points.rdd());

        LogisticRegressionModel model = LogisticRegressionWithSGD.train(
                points.rdd(), iterations, stepSize);

        System.out.print("Final w: " + model.weights());

        sc.stop();
    }
}

Related Tutorials