List of usage examples for weka.filters.unsupervised.attribute StringToWordVector setWordsToKeep
public void setWordsToKeep(int newWordsToKeep)
From source file:com.hack23.cia.service.impl.action.user.wordcount.WordCounterImpl.java
License:Apache License
@Override public Map<String, Integer> calculateWordCount(final DocumentContentData documentContentData, final int maxResult) { final String html = documentContentData.getContent(); final Attribute input = new Attribute("html", (ArrayList<String>) null); final ArrayList<Attribute> inputVec = new ArrayList<>(); inputVec.add(input);/* www . java2 s . c o m*/ final Instances htmlInst = new Instances("html", inputVec, 1); htmlInst.add(new DenseInstance(1)); htmlInst.instance(0).setValue(0, html); final StopwordsHandler StopwordsHandler = new StopwordsHandler() { @Override public boolean isStopword(final String word) { return word.length() < 5; } }; final NGramTokenizer tokenizer = new NGramTokenizer(); tokenizer.setNGramMinSize(1); tokenizer.setNGramMaxSize(1); tokenizer.setDelimiters(" \r\n\t.,;:'\"()?!'"); final StringToWordVector filter = new StringToWordVector(); filter.setTokenizer(tokenizer); filter.setStopwordsHandler(StopwordsHandler); filter.setLowerCaseTokens(true); filter.setOutputWordCounts(true); filter.setWordsToKeep(maxResult); final Map<String, Integer> result = new HashMap<>(); try { filter.setInputFormat(htmlInst); final Instances dataFiltered = Filter.useFilter(htmlInst, filter); final Instance last = dataFiltered.lastInstance(); final int numAttributes = last.numAttributes(); for (int i = 0; i < numAttributes; i++) { result.put(last.attribute(i).name(), Integer.valueOf(last.toString(i))); } } catch (final Exception e) { LOGGER.warn("Problem calculating wordcount for : {} , exception:{}", documentContentData.getId(), e); } return result; }
From source file:com.ivanrf.smsspam.SpamClassifier.java
License:Apache License
private static FilteredClassifier initFilterClassifier(int wordsToKeep, String tokenizerOp, boolean useAttributeSelection, String classifierOp, boolean boosting) throws Exception { StringToWordVector filter = new StringToWordVector(); filter.setDoNotOperateOnPerClassBasis(true); filter.setLowerCaseTokens(true);//w w w . j a v a2s .c om filter.setWordsToKeep(wordsToKeep); if (!tokenizerOp.equals(TOKENIZER_DEFAULT)) { //Make a tokenizer WordTokenizer wt = new WordTokenizer(); if (tokenizerOp.equals(TOKENIZER_COMPLETE)) wt.setDelimiters(" \r\n\t.,;:\'\"()?!-+*&#$%/=<>[]_`@\\^{}"); else //TOKENIZER_COMPLETE_NUMBERS) wt.setDelimiters(" \r\n\t.,;:\'\"()?!-+*&#$%/=<>[]_`@\\^{}|~0123456789"); filter.setTokenizer(wt); } FilteredClassifier classifier = new FilteredClassifier(); classifier.setFilter(filter); if (useAttributeSelection) { AttributeSelection as = new AttributeSelection(); as.setEvaluator(new InfoGainAttributeEval()); Ranker r = new Ranker(); r.setThreshold(0); as.setSearch(r); MultiFilter mf = new MultiFilter(); mf.setFilters(new Filter[] { filter, as }); classifier.setFilter(mf); } if (classifierOp.equals(CLASSIFIER_SMO)) classifier.setClassifier(new SMO()); else if (classifierOp.equals(CLASSIFIER_NB)) classifier.setClassifier(new NaiveBayes()); else if (classifierOp.equals(CLASSIFIER_IB1)) classifier.setClassifier(new IBk(1)); else if (classifierOp.equals(CLASSIFIER_IB3)) classifier.setClassifier(new IBk(3)); else if (classifierOp.equals(CLASSIFIER_IB5)) classifier.setClassifier(new IBk(5)); else if (classifierOp.equals(CLASSIFIER_PART)) classifier.setClassifier(new PART()); //Tarda mucho if (boosting) { AdaBoostM1 boost = new AdaBoostM1(); boost.setClassifier(classifier.getClassifier()); classifier.setClassifier(boost); //Con NB tarda mucho } return classifier; }
From source file:newsclassifier.NewsClassifier.java
public void StrToWV(String sFile) throws Exception { StringToWordVector filter = new StringToWordVector(); filter.setInputFormat(data);//from www. j a va 2 s . co m /*filter.setIDFTransform(false); filter.setTFTransform(true); filter.setAttributeIndices("1-2"); //attributenameprefix filter.setDoNotOperateOnPerClassBasis(true); filter.setInvertSelection(false); filter.setLowerCaseTokens(true); filter.setMinTermFreq(1); //filter.setNormalizeDocLength(true); filter.setOutputWordCounts(false); //filter.setPeriodicPruning(-1); //filter.setStemmer(null); filter.setStopwords(new File(sFile));*/ //String[] opts = weka.core.Utils.splitOptions("-tokenizer \"weka.core.tokenizers.WordTokenizer -delimiters \\\" \\\\r \\\\t.,;:\\\\\\'\\\\\\\"()?!1234567890 `~!@#\\\\\\%^&*[]-_+={}\\\\\\\\/|?>< \\\\r\\\\t\\\"\""); String[] opts = weka.core.Utils.splitOptions( "-R 1-2 -W 3000 -prune-rate -1.0 -T -N 0 -L -S -stemmer weka.core.stemmers.NullStemmer -M 1 -O -stopwords \"C:\\\\Users\\\\USER\\\\Dropbox\\\\Works\\\\IF\\\\AI\\\\Tubes 2\\\\s.txt\" -tokenizer \"weka.core.tokenizers.WordTokenizer -delimiters \\\" \\\\t.,;:\\\\\\'\\\\\\\"()?!1234567890 `~!@#\\\\\\%^&*[]-_+={}\\\\\\\\/|?>< \\\\t\\\"\""); filter.setOptions(opts); //belum pake delimiter!! filter.setWordsToKeep(3000); data = Filter.useFilter(data, filter); //return newData; //data = newData; }
From source file:nlpmusic.StringClusterer.java
public ArrayList<ArrayList<String>> cluster(ArrayList<String> tem) throws Exception { Instances source = listLoad(tem);/* ww w . ja v a 2s .co m*/ StringToWordVector vect = new StringToWordVector(); vect.setWordsToKeep(to_keep); vect.setInputFormat(source); Instances datas = Filter.useFilter(source, vect); //vect.setDoNotOperateOnPerClassBasis(true); //System.out.println("ASDASD" + vect.wordsToKeepTipText()); //System.out.println(datas.numAttributes()); //System.out.println("ASDASD" + vect.getWordsToKeep()); DBSCAN clusterer = new DBSCAN(); clusterer.setEpsilon(threshold); clusterer.setMinPoints(min_points); clusterer.buildClusterer(datas); ArrayList<ArrayList<String>> ret = new ArrayList<>(); for (int i = 0; i < clusterer.numberOfClusters(); i++) { ArrayList<String> to_add = new ArrayList<>(); //System.out.println(i); for (int j = 0; j < datas.size(); j++) { try { if (clusterer.clusterInstance(datas.get(j)) == i) //System.out.println("* " + source.get(j).toString() + " *"); to_add.add(source.get(j).toString()); } catch (Exception e) { //e.printStackTrace(); } } ret.add(to_add); } return ret; }
From source file:org.montp2.m1decol.ter.gramms.filters.FilterTokenizerBoolean.java
License:Open Source License
public void indexingToTokenizer(String inPath, String outPath) throws Exception { WordTokenizer wordTokenizer = new WordTokenizer(); wordTokenizer.setDelimiters("\r \t.,;:'\"()?!"); Instances inputInstances = WekaUtils.loadARFF(inPath); StringToWordVector filter = new StringToWordVector(); filter.setInputFormat(inputInstances); filter.setDoNotOperateOnPerClassBasis(false); filter.setInvertSelection(false);//from w w w . j a v a 2 s.c o m filter.setLowerCaseTokens(true); filter.setOutputWordCounts(false); filter.setTokenizer(wordTokenizer); filter.setUseStoplist(true); filter.setWordsToKeep(wordsTokeep); Instances outputInstances = Filter.useFilter(inputInstances, filter); OutputStreamUtils.writeSimple(outputInstances.toString(), outPath); }
From source file:org.montp2.m1decol.ter.gramms.filters.FilterTokenizerIDFT.java
License:Open Source License
public void indexingToTokenizer(String inPath, String outPath) throws Exception { WordTokenizer wordTokenizer = new WordTokenizer(); wordTokenizer.setDelimiters("\r \t.,;:'\"()?!"); Instances inputInstances = WekaUtils.loadARFF(inPath); StringToWordVector filter = new StringToWordVector(); filter.setInputFormat(inputInstances); filter.setIDFTransform(true);/* ww w .ja va2 s .c om*/ filter.setTFTransform(true); filter.setDoNotOperateOnPerClassBasis(false); filter.setInvertSelection(false); filter.setLowerCaseTokens(true); filter.setMinTermFreq(3); filter.setOutputWordCounts(true); filter.setTokenizer(wordTokenizer); filter.setUseStoplist(true); filter.setWordsToKeep(200); Instances outputInstances = Filter.useFilter(inputInstances, filter); OutputStreamUtils.writeSimple(outputInstances.toString(), outPath); }
From source file:org.montp2.m1decol.ter.gramms.filters.FilterTokenizerVector.java
License:Open Source License
public void indexingToTokenizer(String inPath, String outPath) throws Exception { WordTokenizer wordTokenizer = new WordTokenizer(); wordTokenizer.setDelimiters("\r \t.,;:'\"()?!"); Instances inputInstances = WekaUtils.loadARFF(inPath); StringToWordVector filter = new StringToWordVector(); filter.setInputFormat(inputInstances); filter.setDoNotOperateOnPerClassBasis(false); filter.setInvertSelection(false);// www .j a va 2 s .com filter.setLowerCaseTokens(true); filter.setMinTermFreq(3); filter.setOutputWordCounts(true); filter.setTokenizer(wordTokenizer); filter.setUseStoplist(true); filter.setWordsToKeep(200); Instances outputInstances = Filter.useFilter(inputInstances, filter); OutputStreamUtils.writeSimple(outputInstances.toString(), outPath); }