Java tutorial
/******************************************************************************* * Copyright 2012 University of Southern California * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * This code was developed by the Information Integration Group as part * of the Karma project at the Information Sciences Institute of the * University of Southern California. For more information, publications, * and related projects, please see: http://www.isi.edu/integration ******************************************************************************/ package edu.isi.karma.cleaning.features; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Vector; import org.apache.mahout.classifier.sgd.CsvRecordFactory; import org.apache.mahout.classifier.sgd.OnlineLogisticRegression; import org.apache.mahout.classifier.sgd.RecordFactory; import org.apache.mahout.math.DenseVector; import org.apache.mahout.math.RandomAccessSparseVector; import com.google.common.collect.Lists; import com.google.common.io.Closeables; import edu.isi.karma.cleaning.PartitionClassifierType; public class RecordClassifier2 implements PartitionClassifierType { HashMap<String, Vector<String>> trainData = new HashMap<String, Vector<String>>(); RecordFeatureSet rf = new RecordFeatureSet(); OnlineLogisticRegression cf; List<String> labels = new ArrayList<String>(); LogisticModelParameters lmp; public RecordClassifier2() { } public OnlineLogisticRegression train(HashMap<String, Vector<String>> traindata) throws Exception { String csvTrainFile = "./target/tmp/csvtrain.csv"; Data2Features.Traindata2CSV(traindata, csvTrainFile, rf); lmp = new LogisticModelParameters(); lmp.setTargetVariable("label"); lmp.setMaxTargetCategories(rf.labels.size()); lmp.setNumFeatures(rf.getFeatureNames().size()); List<String> typeList = Lists.newArrayList(); typeList.add("numeric"); List<String> predictorList = Lists.newArrayList(); for (String attr : rf.getFeatureNames()) { if (attr.compareTo("lable") != 0) { predictorList.add(attr); } } lmp.setTypeMap(predictorList, typeList); // lmp.setUseBias(!getBooleanArgument(cmdLine, noBias)); // lmp.setTypeMap(predictorList, typeList); lmp.setLambda(1e-4); lmp.setLearningRate(50); int passes = 100; CsvRecordFactory csv = lmp.getCsvRecordFactory(); OnlineLogisticRegression lr = lmp.createRegression(); for (int pass = 0; pass < passes; pass++) { BufferedReader in = new BufferedReader(new FileReader(new File(csvTrainFile))); ; try { // read variable names csv.firstLine(in.readLine()); String line = in.readLine(); while (line != null) { // for each new line, get target and predictors RandomAccessSparseVector input = new RandomAccessSparseVector(lmp.getNumFeatures()); int targetValue = csv.processLine(line, input); String label = csv.getTargetCategories().get(lr.classifyFull(input).maxValueIndex()); // now update model lr.train(targetValue, input); line = in.readLine(); } } finally { Closeables.closeQuietly(in); } } labels = csv.getTargetCategories(); return lr; } private static double predictorWeight(OnlineLogisticRegression lr, int row, RecordFactory csv, String predictor) { double weight = 0; for (Integer column : csv.getTraceDictionary().get(predictor)) { weight += lr.getBeta().get(row, column); } return weight; } public String Classify(String instance) { Collection<Feature> cfeat = rf.computeFeatures(instance, ""); Feature[] x = cfeat.toArray(new Feature[cfeat.size()]); // row.add(f.getName()); RandomAccessSparseVector row = new RandomAccessSparseVector(x.length); String line = ""; for (int k = 0; k < cfeat.size(); k++) { line += x[k].getScore() + ","; } line += "label"; // dummy class label for testing CsvRecordFactory csv = lmp.getCsvRecordFactory(); csv.processLine(line, row); DenseVector dvec = (DenseVector) this.cf.classifyFull(row); String label = labels.get(dvec.maxValueIndex()); return label; } @Override public void addTrainingData(String value, String label) { if (trainData.containsKey(label)) { trainData.get(label).add(value); } else { Vector<String> vsStrings = new Vector<String>(); vsStrings.add(value); trainData.put(label, vsStrings); } } @Override public String learnClassifer() { try { this.cf = this.train(trainData); } catch (Exception e) { System.out.println(""); } return this.cf.toString(); } @Override public String getLabel(String value) { try { String label = this.Classify(value); if (label.length() > 0) return label; else { return "null_in_classification"; } } catch (Exception e) { return "null_in_classification"; // TODO: handle exception } } public static void main(String[] args) { try { HashMap<String, Vector<String>> trainData = new HashMap<String, Vector<String>>(); Vector<String> test = new Vector<String>(); Vector<String> par1 = new Vector<String>(); par1.add("1286 adams blvd"); par1.add("3711 catalina st"); // par1.add("11 w 37th pl, los angeles"); Vector<String> par2 = new Vector<String>(); par2.add("1142 37st"); // par2.add("1 jefferson st"); Vector<String> par3 = new Vector<String>(); par3.add("710 27"); trainData.put("c1", par1); trainData.put("c2", par2); trainData.put("c3", par3); test.add("2353 portland st"); RecordClassifier2 rc = new RecordClassifier2(); for (String key : trainData.keySet()) { for (String value : trainData.get(key)) { rc.addTrainingData(value, key); } } rc.learnClassifer(); System.out.println(rc.Classify(test.get(0))); } catch (Exception ex) { ex.printStackTrace(); } } }