List of usage examples for org.apache.hadoop.io DefaultStringifier fromString
@Override
public T fromString(String str) throws IOException
From source file:org.apache.mahout.classifier.cbayes.CBayesThetaNormalizerDriver.java
License:Apache License
/** * Run the job//from w w w . j a va2s. c om * * @param input the input pathname String * @param output the output pathname String */ public static void runJob(String input, String output) throws IOException { JobClient client = new JobClient(); JobConf conf = new JobConf(CBayesThetaNormalizerDriver.class); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(DoubleWritable.class); FileInputFormat.addInputPath(conf, new Path(output + "/trainer-weights/Sigma_j")); FileInputFormat.addInputPath(conf, new Path(output + "/trainer-tfIdf/trainer-tfIdf")); Path outPath = new Path(output + "/trainer-thetaNormalizer"); FileOutputFormat.setOutputPath(conf, outPath); conf.setNumMapTasks(100); //conf.setNumReduceTasks(1); conf.setMapperClass(CBayesThetaNormalizerMapper.class); conf.setInputFormat(SequenceFileInputFormat.class); conf.setCombinerClass(CBayesThetaNormalizerReducer.class); conf.setReducerClass(CBayesThetaNormalizerReducer.class); conf.setOutputFormat(SequenceFileOutputFormat.class); conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization"); // Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code FileSystem dfs = FileSystem.get(outPath.toUri(), conf); if (dfs.exists(outPath)) { dfs.delete(outPath, true); } Path Sigma_kFiles = new Path(output + "/trainer-weights/Sigma_k/*"); Map<String, Double> labelWeightSum = SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf); DefaultStringifier<Map<String, Double>> mapStringifier = new DefaultStringifier<Map<String, Double>>(conf, GenericsUtil.getClass(labelWeightSum)); String labelWeightSumString = mapStringifier.toString(labelWeightSum); log.info("Sigma_k for Each Label"); Map<String, Double> c = mapStringifier.fromString(labelWeightSumString); log.info("{}", c); conf.set("cnaivebayes.sigma_k", labelWeightSumString); Path sigma_kSigma_jFile = new Path(output + "/trainer-weights/Sigma_kSigma_j/*"); double sigma_jSigma_k = SequenceFileModelReader.readSigma_jSigma_k(dfs, sigma_kSigma_jFile, conf); DefaultStringifier<Double> stringifier = new DefaultStringifier<Double>(conf, Double.class); String sigma_jSigma_kString = stringifier.toString(sigma_jSigma_k); log.info("Sigma_kSigma_j for each Label and for each Features"); double retSigma_jSigma_k = stringifier.fromString(sigma_jSigma_kString); log.info("{}", retSigma_jSigma_k); conf.set("cnaivebayes.sigma_jSigma_k", sigma_jSigma_kString); Path vocabCountFile = new Path(output + "/trainer-tfIdf/trainer-vocabCount/*"); double vocabCount = SequenceFileModelReader.readVocabCount(dfs, vocabCountFile, conf); String vocabCountString = stringifier.toString(vocabCount); log.info("Vocabulary Count"); conf.set("cnaivebayes.vocabCount", vocabCountString); double retvocabCount = stringifier.fromString(vocabCountString); log.info("{}", retvocabCount); client.setConf(conf); JobClient.runJob(conf); }
From source file:org.apache.mahout.common.Parameters.java
License:Apache License
public static Map<String, String> parseParams(String serializedString) throws IOException { Configuration conf = new Configuration(); conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); Map<String, String> params = Maps.newHashMap(); DefaultStringifier<Map<String, String>> mapStringifier = new DefaultStringifier<Map<String, String>>(conf, GenericsUtil.getClass(params)); return mapStringifier.fromString(serializedString); }
From source file:org.apache.mahout.fpm.pfpgrowth.PFPGrowth.java
License:Apache License
/** * Generates the fList from the serialized string representation * /*ww w . ja v a 2s. c o m*/ * @param params * @param key * @param conf * @return Deserialized Feature Frequency List * @throws IOException */ public static List<Pair<String, Long>> deserializeList(Parameters params, String key, Configuration conf) throws IOException { List<Pair<String, Long>> list = new ArrayList<Pair<String, Long>>(); conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); DefaultStringifier<List<Pair<String, Long>>> listStringifier = new DefaultStringifier<List<Pair<String, Long>>>( conf, GenericsUtil.getClass(list)); String serializedString = params.get(key, listStringifier.toString(list)); list = listStringifier.fromString(serializedString); return list; }
From source file:org.apache.mahout.fpm.pfpgrowth.PFPGrowth.java
License:Apache License
/** * Generates the gList(Group ID Mapping of Various frequent Features) Map from the corresponding serialized * representation/*from w ww .ja v a 2 s .co m*/ * * @param params * @param key * @param conf * @return Deserialized Group List * @throws IOException */ public static Map<String, Long> deserializeMap(Parameters params, String key, Configuration conf) throws IOException { Map<String, Long> map = new HashMap<String, Long>(); conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); DefaultStringifier<Map<String, Long>> mapStringifier = new DefaultStringifier<Map<String, Long>>(conf, GenericsUtil.getClass(map)); String gListString = params.get(key, mapStringifier.toString(map)); map = mapStringifier.fromString(gListString); return map; }
From source file:org.apache.mahout.text.wikipedia.WikipediaDatasetCreatorMapper.java
License:Apache License
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); if (inputCategories == null) { Set<String> newCategories = Sets.newHashSet(); DefaultStringifier<Set<String>> setStringifier = new DefaultStringifier<Set<String>>(conf, GenericsUtil.getClass(newCategories)); String categoriesStr = conf.get("wikipedia.categories", setStringifier.toString(newCategories)); Set<String> inputCategoriesSet = setStringifier.fromString(categoriesStr); inputCategories = Lists.newArrayList(inputCategoriesSet); inputCategoryPatterns = Lists.newArrayListWithCapacity(inputCategories.size()); for (String inputCategory : inputCategories) { inputCategoryPatterns.add(Pattern.compile(".*\\b" + inputCategory + "\\b.*")); }/*from w w w .jav a 2 s . co m*/ } exactMatchOnly = conf.getBoolean("exact.match.only", false); if (analyzer == null) { String analyzerStr = conf.get("analyzer.class", WikipediaAnalyzer.class.getName()); analyzer = ClassUtils.instantiateAs(analyzerStr, Analyzer.class); } log.info("Configure: Input Categories size: {} Exact Match: {} Analyzer: {}", inputCategories.size(), exactMatchOnly, analyzer.getClass().getName()); }
From source file:org.apache.mahout.text.wikipedia.WikipediaMapper.java
License:Apache License
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); Set<String> newCategories = Sets.newHashSet(); DefaultStringifier<Set<String>> setStringifier = new DefaultStringifier<Set<String>>(conf, GenericsUtil.getClass(newCategories)); String categoriesStr = conf.get("wikipedia.categories"); inputCategories = setStringifier.fromString(categoriesStr); exactMatchOnly = conf.getBoolean("exact.match.only", false); all = conf.getBoolean("all.files", false); removeLabels = conf.getBoolean("remove.labels", false); log.info("Configure: Input Categories size: {} All: {} Exact Match: {} Remove Labels from Text: {}", inputCategories.size(), all, exactMatchOnly, removeLabels); }
From source file:org.apache.mahout.text.WikipediaMapper.java
License:Apache License
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); try {// w w w . ja v a2 s .c om if (inputCategories == null) { Set<String> newCategories = new HashSet<String>(); DefaultStringifier<Set<String>> setStringifier = new DefaultStringifier<Set<String>>(conf, GenericsUtil.getClass(newCategories)); String categoriesStr = setStringifier.toString(newCategories); categoriesStr = conf.get("wikipedia.categories", categoriesStr); inputCategories = setStringifier.fromString(categoriesStr); } exactMatchOnly = conf.getBoolean("exact.match.only", false); all = conf.getBoolean("all.files", true); } catch (IOException ex) { throw new IllegalStateException(ex); } log.info("Configure: Input Categories size: {} All: {} Exact Match: {}", new Object[] { inputCategories.size(), all, exactMatchOnly }); }
From source file:redpoll.clusterer.kmeans.KMeansMapper.java
License:Apache License
@Override public void configure(JobConf job) { Cluster.configure(job);/*w ww .j a v a 2s . com*/ try { HashMap<String, WritableVector> centers = new HashMap<String, WritableVector>(); DefaultStringifier<HashMap<String, WritableVector>> stringifier = new DefaultStringifier<HashMap<String, WritableVector>>( job, GenericsUtil.getClass(centers)); String centersString = job.get("redpoll.clusterer.kmeans.centers"); centers = stringifier.fromString(centersString); for (Map.Entry<String, WritableVector> entry : centers.entrySet()) { clusters.add(new Cluster(entry.getKey(), entry.getValue())); } } catch (IOException exp) { exp.printStackTrace(); } }
From source file:redpoll.clusterer.kmeans.KMeansReducer.java
License:Apache License
@Override public void configure(JobConf job) { Cluster.configure(job);// ww w . j a v a 2s. c o m try { HashMap<String, WritableVector> centers = new HashMap<String, WritableVector>(); DefaultStringifier<HashMap<String, WritableVector>> stringifier = new DefaultStringifier<HashMap<String, WritableVector>>( job, GenericsUtil.getClass(centers)); String centersString = job.get("redpoll.clusterer.kmeans.centers"); centers = stringifier.fromString(centersString); for (Map.Entry<String, WritableVector> entry : centers.entrySet()) { Cluster cluster = new Cluster(entry.getKey(), entry.getValue()); cluster.addPoint(cluster.getCenter()); clusters.put(entry.getKey(), cluster); } } catch (IOException exp) { exp.printStackTrace(); } }
From source file:redpoll.text.TfIdfMapper.java
License:Apache License
@Override public void configure(JobConf job) { try {/*from w w w . j a v a2 s. co m*/ terms = new HashMap<String, Integer>(); DefaultStringifier<HashMap<String, Integer>> mapStringifier = new DefaultStringifier<HashMap<String, Integer>>( job, GenericsUtil.getClass(terms)); String docFreqString = job.get("redpoll.text.terms"); terms = mapStringifier.fromString(docFreqString); toalDocNum = job.getInt("redpoll.docs.num", 1024); } catch (IOException exp) { exp.printStackTrace(); } }