Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package imageClassify; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Random; import java.util.Scanner; import org.apache.commons.cli2.CommandLine; import org.apache.commons.cli2.Group; import org.apache.commons.cli2.Option; import org.apache.commons.cli2.OptionException; import org.apache.commons.cli2.builder.ArgumentBuilder; import org.apache.commons.cli2.builder.DefaultOptionBuilder; import org.apache.commons.cli2.builder.GroupBuilder; import org.apache.commons.cli2.commandline.Parser; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.mahout.classifier.ClassifierResult; import org.apache.mahout.classifier.RegressionResultAnalyzer; import org.apache.mahout.classifier.ResultAnalyzer; import org.apache.mahout.classifier.df.DFUtils; import org.apache.mahout.classifier.df.DecisionForest; import org.apache.mahout.classifier.df.data.DataConverter; import org.apache.mahout.classifier.df.data.Dataset; import org.apache.mahout.classifier.df.data.Instance; import org.apache.mahout.classifier.df.mapreduce.Classifier; import org.apache.mahout.common.CommandLineUtil; import org.apache.mahout.common.RandomUtils; import org.apache.mahout.common.commandline.DefaultOptionCreator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.io.Closeables; /** * Tool to classify a Dataset using a previously built Decision Forest */ public class TestForest extends Configured implements Tool { private static final Logger log = LoggerFactory.getLogger(TestForest.class); private FileSystem dataFS; private Path dataPath; // test data path private Path datasetPath; private Path modelPath; // path where the forest is stored private FileSystem outFS; private Path outputPath; // path to predictions file, if null do not output the predictions private boolean analyze; // analyze the classification results ? private boolean useMapreduce; // use the mapreduce classifier ? @Override public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException { DefaultOptionBuilder obuilder = new DefaultOptionBuilder(); ArgumentBuilder abuilder = new ArgumentBuilder(); GroupBuilder gbuilder = new GroupBuilder(); Option inputOpt = DefaultOptionCreator.inputOption().create(); Option datasetOpt = obuilder.withLongName("dataset").withShortName("ds").withRequired(true) .withArgument(abuilder.withName("dataset").withMinimum(1).withMaximum(1).create()) .withDescription("Dataset path").create(); Option modelOpt = obuilder.withLongName("model").withShortName("m").withRequired(true) .withArgument(abuilder.withName("path").withMinimum(1).withMaximum(1).create()) .withDescription("Path to the Decision Forest").create(); Option outputOpt = DefaultOptionCreator.outputOption().create(); Option analyzeOpt = obuilder.withLongName("analyze").withShortName("a").withRequired(false).create(); Option mrOpt = obuilder.withLongName("mapreduce").withShortName("mr").withRequired(false).create(); Option helpOpt = DefaultOptionCreator.helpOption(); Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(datasetOpt).withOption(modelOpt) .withOption(outputOpt).withOption(analyzeOpt).withOption(mrOpt).withOption(helpOpt).create(); try { Parser parser = new Parser(); parser.setGroup(group); CommandLine cmdLine = parser.parse(args); if (cmdLine.hasOption("help")) { CommandLineUtil.printHelp(group); return -1; } String dataName = cmdLine.getValue(inputOpt).toString(); String datasetName = cmdLine.getValue(datasetOpt).toString(); String modelName = cmdLine.getValue(modelOpt).toString(); String outputName = cmdLine.hasOption(outputOpt) ? cmdLine.getValue(outputOpt).toString() : null; analyze = cmdLine.hasOption(analyzeOpt); useMapreduce = cmdLine.hasOption(mrOpt); if (log.isDebugEnabled()) { log.debug("inout : {}", dataName); log.debug("dataset : {}", datasetName); log.debug("model : {}", modelName); log.debug("output : {}", outputName); log.debug("analyze : {}", analyze); log.debug("mapreduce : {}", useMapreduce); } dataPath = new Path(dataName); datasetPath = new Path(datasetName); modelPath = new Path(modelName); if (outputName != null) { outputPath = new Path(outputName); } } catch (OptionException e) { log.warn(e.toString(), e); CommandLineUtil.printHelp(group); return -1; } testForest(); return 0; } private void testForest() throws IOException, ClassNotFoundException, InterruptedException { // make sure the output file does not exist if (outputPath != null) { outFS = outputPath.getFileSystem(getConf()); if (outFS.exists(outputPath)) { throw new IllegalArgumentException("Output path already exists"); } } // make sure the decision forest exists FileSystem mfs = modelPath.getFileSystem(getConf()); if (!mfs.exists(modelPath)) { throw new IllegalArgumentException("The forest path does not exist"); } // make sure the test data exists dataFS = dataPath.getFileSystem(getConf()); if (!dataFS.exists(dataPath)) { throw new IllegalArgumentException("The Test data path does not exist"); } if (useMapreduce) { mapreduce(); } else { sequential(); } } private void mapreduce() throws ClassNotFoundException, IOException, InterruptedException { if (outputPath == null) { throw new IllegalArgumentException( "You must specify the ouputPath when using the mapreduce implementation"); } Classifier classifier = new Classifier(modelPath, dataPath, datasetPath, outputPath, getConf()); classifier.run(); if (analyze) { double[][] results = classifier.getResults(); if (results != null) { Dataset dataset = Dataset.load(getConf(), datasetPath); if (dataset.isNumerical(dataset.getLabelId())) { RegressionResultAnalyzer regressionAnalyzer = new RegressionResultAnalyzer(); regressionAnalyzer.setInstances(results); log.info("{}", regressionAnalyzer); } else { ResultAnalyzer analyzer = new ResultAnalyzer(Arrays.asList(dataset.labels()), "unknown"); for (double[] res : results) { analyzer.addInstance(dataset.getLabelString(res[0]), new ClassifierResult(dataset.getLabelString(res[1]), 1.0)); } log.info("{}", analyzer); } } } } private void sequential() throws IOException { log.info("Loading the forest..."); DecisionForest forest = DecisionForest.load(getConf(), modelPath); if (forest == null) { log.error("No Decision Forest found!"); return; } // load the dataset Dataset dataset = Dataset.load(getConf(), datasetPath); DataConverter converter = new DataConverter(dataset); log.info("Sequential classification..."); long time = System.currentTimeMillis(); Random rng = RandomUtils.getRandom(); List<double[]> resList = Lists.newArrayList(); if (dataFS.getFileStatus(dataPath).isDir()) { //the input is a directory of files testDirectory(outputPath, converter, forest, dataset, resList, rng); } else { // the input is one single file testFile(dataPath, outputPath, converter, forest, dataset, resList, rng); } time = System.currentTimeMillis() - time; log.info("Classification Time: {}", DFUtils.elapsedTime(time)); if (analyze) { if (dataset.isNumerical(dataset.getLabelId())) { RegressionResultAnalyzer regressionAnalyzer = new RegressionResultAnalyzer(); double[][] results = new double[resList.size()][2]; regressionAnalyzer.setInstances(resList.toArray(results)); log.info("{}", regressionAnalyzer); } else { ResultAnalyzer analyzer = new ResultAnalyzer(Arrays.asList(dataset.labels()), "unknown"); for (double[] r : resList) { analyzer.addInstance(dataset.getLabelString(r[0]), new ClassifierResult(dataset.getLabelString(r[1]), 1.0)); } log.info("{}", analyzer); } } } private void testDirectory(Path outPath, DataConverter converter, DecisionForest forest, Dataset dataset, Collection<double[]> results, Random rng) throws IOException { Path[] infiles = DFUtils.listOutputFiles(dataFS, dataPath); for (Path path : infiles) { log.info("Classifying : {}", path); Path outfile = outPath != null ? new Path(outPath, path.getName()).suffix(".out") : null; testFile(path, outfile, converter, forest, dataset, results, rng); } } private void testFile(Path inPath, Path outPath, DataConverter converter, DecisionForest forest, Dataset dataset, Collection<double[]> results, Random rng) throws IOException { // create the predictions file FSDataOutputStream ofile = null; if (outPath != null) { ofile = outFS.create(outPath); } FSDataInputStream input = dataFS.open(inPath); try { Scanner scanner = new Scanner(input, "UTF-8"); while (scanner.hasNextLine()) { String line = scanner.nextLine(); if (line.isEmpty()) { continue; // skip empty lines } Instance instance = converter.convert(line); double prediction = forest.classify(dataset, rng, instance); if (ofile != null) { ofile.writeChars(Double.toString(prediction)); // write the prediction ofile.writeChar('\n'); } results.add(new double[] { dataset.getLabel(instance), prediction }); } scanner.close(); } finally { Closeables.close(input, true); } } public static void main(String[] args) throws Exception { ToolRunner.run(new Configuration(), new TestForest(), args); } }