List of usage examples for org.apache.mahout.cf.taste.impl.common FastByIDMap FastByIDMap
public FastByIDMap(int size)
From source file:com.anjuke.romar.mahout.model.RomarMemoryIDMigrator.java
License:Apache License
public RomarMemoryIDMigrator() { this._longToString = new FastByIDMap<String>(INIT_SIZE); }
From source file:com.recsys.factorizer.CustomAbstractFactorizer.java
License:Apache License
private static FastByIDMap<Integer> createIDMapping(int size, LongPrimitiveIterator idIterator) { FastByIDMap<Integer> mapping = new FastByIDMap<Integer>(size); int index = 0; while (idIterator.hasNext()) { mapping.put(idIterator.nextLong(), index++); }//from ww w.j av a2 s . c om return mapping; }
From source file:com.webir.popcornsaver.cluster.TreeClusteringRecommender.java
License:Apache License
private static FastByIDMap<FastIDSet> computeClustersPerUserID(Collection<FastIDSet> clusters) { FastByIDMap<FastIDSet> clustersPerUser = new FastByIDMap<FastIDSet>(clusters.size()); for (FastIDSet cluster : clusters) { LongPrimitiveIterator it = cluster.iterator(); while (it.hasNext()) { clustersPerUser.put(it.next(), cluster); }//from ww w . j a va 2s . co m } return clustersPerUser; }
From source file:io.seldon.clustering.recommender.MemoryUserClusterStore.java
License:Apache License
public MemoryUserClusterStore(String client, int entries) { logger.info("MemoryUserClusterStore for " + client + " of size " + entries); this.store = new FastByIDMap<>(entries); this.client = client; this.clusterGroups = new ConcurrentHashMap<>(); this.transientClusters = new ConcurrentHashMap<>(); }
From source file:lib.eval.AbstractRecommenderEvaluator.java
License:Apache License
@Override public double evaluate(RecommenderBuilder recommenderBuilder, DataModelBuilder dataModelBuilder, DataModel dataModel, double trainingPercentage, double evaluationPercentage) throws TasteException { Preconditions.checkArgument(recommenderBuilder != null, "recommenderBuilder is null"); Preconditions.checkArgument(dataModel != null, "dataModel is null"); Preconditions.checkArgument(trainingPercentage >= 0.0 && trainingPercentage <= 1.0, "Invalid trainingPercentage: " + trainingPercentage); Preconditions.checkArgument(evaluationPercentage >= 0.0 && evaluationPercentage <= 1.0, "Invalid evaluationPercentage: " + evaluationPercentage); log.info("Beginning evaluation using {} of {}", trainingPercentage, dataModel); int numUsers = dataModel.getNumUsers(); FastByIDMap<PreferenceArray> trainingUsers = new FastByIDMap<PreferenceArray>( 1 + (int) (evaluationPercentage * numUsers)); FastByIDMap<PreferenceArray> testUserPrefs = new FastByIDMap<PreferenceArray>( 1 + (int) (evaluationPercentage * numUsers)); LongPrimitiveIterator it = dataModel.getUserIDs(); while (it.hasNext()) { long userID = it.nextLong(); if (random.nextDouble() < evaluationPercentage) { processOneUser(trainingPercentage, trainingUsers, testUserPrefs, userID, dataModel); } else { //this user will not be used for evaluation so use all its preferences in training PreferenceArray trainingPrefs = dataModel.getPreferencesFromUser(userID); trainingUsers.put(userID, trainingPrefs); }//from ww w. ja v a 2 s .c o m } DataModel trainingModel = dataModelBuilder == null ? new GenericDataModel(trainingUsers) : dataModelBuilder.buildDataModel(trainingUsers); Recommender recommender = recommenderBuilder.buildRecommender(trainingModel); double result = getEvaluation(testUserPrefs, recommender); log.info("Evaluation result: {}", result); return result; }
From source file:norbert.mynemo.core.evaluation.PersonnalRecommenderEvaluator.java
License:Apache License
/** * Build the training preferences with a given percentage of user. The preferences are extracted * from the given data model. The returned map does not include the target user. * * <p>/*from w w w. j a v a 2 s . c om*/ * The given percentage must be between 0 and 1. */ private FastByIDMap<PreferenceArray> buildBaseTrainingPreferences(DataModel model, double evaluationPercentage) throws TasteException { FastByIDMap<PreferenceArray> result = new FastByIDMap<PreferenceArray>(model.getNumUsers() - 1); LongPrimitiveIterator it = model.getUserIDs(); while (it.hasNext()) { long userId = it.nextLong(); double nextDouble = random.nextDouble(); if (nextDouble < evaluationPercentage && userId != targetUser) { copyUserPreferences(model, result, userId); } } return result; }
From source file:org.easyrec.mahout.model.EasyrecInMemoryDataModel.java
License:Open Source License
private void intializeDelegate() { try {//from ww w . ja v a 2s . c o m //iterate over all user ids LongPrimitiveIterator it = easyrecDataModelDelegate.getUserIDs(); FastByIDMap<PreferenceArray> preferences = new FastByIDMap<PreferenceArray>( easyrecDataModelDelegate.getNumUsers()); FastByIDMap<FastByIDMap<Long>> timestamps = new FastByIDMap<FastByIDMap<Long>>( easyrecDataModelDelegate.getNumUsers()); while (it.hasNext()) { Long userId = it.next(); //get preferences for each user PreferenceArray prefs = easyrecDataModelDelegate.getPreferencesFromUser(userId); preferences.put(userId, prefs); //get preference times for each user FastByIDMap<Long> timestampsForUser = new FastByIDMap<Long>(prefs.getIDs().length); for (Long itemId : prefs.getIDs()) { timestampsForUser.put(itemId, easyrecDataModelDelegate.getPreferenceTime(userId, itemId)); } timestamps.put(userId, timestampsForUser); } //generate GenericDataModel setMaxPreference(easyrecDataModelDelegate.getMaxPreference()); setMinPreference(easyrecDataModelDelegate.getMinPreference()); this.inMemoryDelegate = new GenericDataModel(preferences, timestamps); } catch (TasteException e) { logger.warn("caught exception while reading preference data", e); } }
From source file:org.plista.kornakapi.core.config.RecommenderConfig.java
License:Apache License
protected void createEmptyFactorization(PersistenceStrategy strategy) throws IOException { strategy.maybePersist(new Factorization(new FastByIDMap<Integer>(0), new FastByIDMap<Integer>(0), new double[0][0], new double[0][0])); }
From source file:org.plista.kornakapi.web.servlets.BaseServlet.java
License:Apache License
private void createEmptyFactorization(PersistenceStrategy strategy) throws IOException { strategy.maybePersist(new Factorization(new FastByIDMap<Integer>(0), new FastByIDMap<Integer>(0), new double[0][0], new double[0][0])); }
From source file:recommender.GenericRecommenderIRStatsEvaluatorCustom.java
License:Apache License
@Override public IRStatistics evaluate(RecommenderBuilder recommenderBuilder, DataModelBuilder dataModelBuilder, DataModel dataModel, IDRescorer rescorer, int at, double relevanceThreshold, double evaluationPercentage) throws TasteException { prop = new Properties(); try {/*from w ww . jav a 2s . co m*/ prop.load(GenericRecommenderIRStatsEvaluatorCustom.class.getResourceAsStream("settings.properties")); } catch (IOException ex) { java.util.logging.Logger.getLogger(GenericRecommenderIRStatsEvaluatorCustom.class.getName()) .log(Level.SEVERE, null, ex); } // load settings from MySQL database loadSettings(); Preconditions.checkArgument(recommenderBuilder != null, "recommenderBuilder is null"); Preconditions.checkArgument(dataModel != null, "dataModel is null"); Preconditions.checkArgument(at >= 1, "at must be at least 1"); Preconditions.checkArgument(evaluationPercentage > 0.0 && evaluationPercentage <= 1.0, "Invalid evaluationPercentage: " + evaluationPercentage + ". Must be: 0.0 < evaluationPercentage <= 1.0"); int numItems = dataModel.getNumItems(); System.out.println("Data model numItems: " + numItems); RunningAverage precision = new FullRunningAverage(); RunningAverage recall = new FullRunningAverage(); RunningAverage fallOut = new FullRunningAverage(); RunningAverage nDCG = new FullRunningAverage(); int numUsersRecommendedFor = 0; int numUsersWithRecommendations = 0; // map to store diversity ranges => number of users HashMap<String, String> map = new HashMap<>(); LongPrimitiveIterator it = dataModel.getUserIDs(); while (it.hasNext()) { long userID = it.nextLong(); if (userID == 0) { continue; } // get the top users if (!list.contains(userID + "")) { continue; } if (random.nextDouble() >= evaluationPercentage) { // Skipped continue; } long start = System.currentTimeMillis(); PreferenceArray prefs = dataModel.getPreferencesFromUser(userID); System.out.println("User preferences: " + prefs); // List some most-preferred items that would count as (most) "relevant" results double theRelevanceThreshold = 0;//Double.isNaN(relevanceThreshold) ? computeThreshold(prefs) : relevanceThreshold; FastIDSet relevantItemIDs = dataSplitter.getRelevantItemsIDs(userID, at, theRelevanceThreshold, dataModel); System.out.println("Relevant items: " + relevantItemIDs); System.out.println("Relevance threshold: " + theRelevanceThreshold); int numRelevantItems = relevantItemIDs.size(); if (numRelevantItems <= 0) { continue; } FastByIDMap<PreferenceArray> trainingUsers = new FastByIDMap<>(dataModel.getNumUsers()); LongPrimitiveIterator it2 = dataModel.getUserIDs(); while (it2.hasNext()) { dataSplitter.processOtherUser(userID, relevantItemIDs, trainingUsers, it2.nextLong(), dataModel); } DataModel trainingModel = dataModelBuilder == null ? new GenericDataModel(trainingUsers) : dataModelBuilder.buildDataModel(trainingUsers); try { trainingModel.getPreferencesFromUser(userID); } catch (NoSuchUserException nsee) { continue; // Oops we excluded all prefs for the user -- just move on } int size = numRelevantItems + trainingModel.getItemIDsFromUser(userID).size(); if (size < 2 * at) { // Really not enough prefs to meaningfully evaluate this user System.out .println("Really not enough prefs (" + size + ") to meaningfully evaluate user: " + userID); continue; } Recommender recommender = recommenderBuilder.buildRecommender(trainingModel); int intersectionSize = 0; List<RecommendedItem> recommendedItems = recommender.recommend(userID, at, rescorer); HashMap<Long, Double> user_preferences = getUserPreferencesList(userID); for (RecommendedItem recommendedItem : recommendedItems) { double preference = isRelevant(user_preferences, recommendedItem); System.out.println("Preference: " + preference); if (relevantItemIDs.contains(recommendedItem.getItemID()) || preference != 0) { intersectionSize++; } } int numRecommendedItems = recommendedItems.size(); // Precision if (numRecommendedItems > 0) { precision.addDatum((double) intersectionSize / (double) numRecommendedItems); System.out.println( "intersectionSize: " + intersectionSize + " numRecommendedItems: " + numRecommendedItems); } // Recall recall.addDatum((double) intersectionSize / (double) numRelevantItems); // Fall-out if (numRelevantItems < size) { fallOut.addDatum( (double) (numRecommendedItems - intersectionSize) / (double) (numItems - numRelevantItems)); } // nDCG // In computing, assume relevant IDs have relevance 1 and others 0 double cumulativeGain = 0.0; double idealizedGain = 0.0; for (int i = 0; i < numRecommendedItems; i++) { RecommendedItem item = recommendedItems.get(i); double discount = 1.0 / log2(i + 2.0); // Classical formulation says log(i+1), but i is 0-based here if (relevantItemIDs.contains(item.getItemID())) { cumulativeGain += discount; } // otherwise we're multiplying discount by relevance 0 so it doesn't do anything // Ideally results would be ordered with all relevant ones first, so this theoretical // ideal list starts with number of relevant items equal to the total number of relevant items if (i < numRelevantItems) { idealizedGain += discount; } } if (idealizedGain > 0.0) { nDCG.addDatum(cumulativeGain / idealizedGain); } // Reach numUsersRecommendedFor++; if (numRecommendedItems > 0) { numUsersWithRecommendations++; } long end = System.currentTimeMillis(); log.info("Evaluated with user {} in {}ms", userID, end - start); log.info("Precision/recall/fall-out/nDCG/reach: {} / {} / {} / {} / {}", precision.getAverage(), recall.getAverage(), fallOut.getAverage(), nDCG.getAverage(), (double) numUsersWithRecommendations / (double) numUsersRecommendedFor); System.out.println("Relevant items: " + numRelevantItems); System.out.println("Precision: " + precision.getAverage()); System.out.println("Recall: " + recall.getAverage()); System.out.println("Fall-out: " + fallOut.getAverage()); System.out.println("nDCG: " + nDCG.getAverage()); System.out.println("Reach: " + (double) numUsersWithRecommendations / (double) numUsersRecommendedFor); double diversity = getDiversity(recommendedItems); System.out.println("Diversity: " + diversity); if (diversity >= 0 && diversity < 0.1) { int count = map.get("0-0.1") != null ? Integer.parseInt(map.get("0-0.1")) + 1 : 1; map.put("0-0.1", count + ""); } else if (diversity >= 0.1 && diversity < 0.2) { int count = map.get("0.1-0.2") != null ? Integer.parseInt(map.get("0.1-0.2")) + 1 : 1; map.put("0.1-0.2", count + ""); } else if (diversity >= 0.2 && diversity < 0.3) { int count = map.get("0.2-0.3") != null ? Integer.parseInt(map.get("0.2-0.3")) + 1 : 1; map.put("0.2-0.3", count + ""); } else if (diversity >= 0.3 && diversity < 0.4) { int count = map.get("0.3-0.4") != null ? Integer.parseInt(map.get("0.3-0.4")) + 1 : 1; map.put("0.3-0.4", count + ""); } else if (diversity >= 0.4 && diversity < 0.5) { int count = map.get("0.4-0.5") != null ? Integer.parseInt(map.get("0.4-0.5")) + 1 : 1; map.put("0.4-0.5", count + ""); } else if (diversity >= 0.5 && diversity < 0.6) { int count = map.get("0.5-0.6") != null ? Integer.parseInt(map.get("0.5-0.6")) + 1 : 1; map.put("0.5-0.6", count + ""); } else if (diversity >= 0.6 && diversity < 0.7) { int count = map.get("0.6-0.7") != null ? Integer.parseInt(map.get("0.6-0.7")) + 1 : 1; map.put("0.6-0.7", count + ""); } else if (diversity >= 0.7 && diversity < 0.8) { int count = map.get("0.7-0.8") != null ? Integer.parseInt(map.get("0.7-0.8")) + 1 : 1; map.put("0.7-0.8", count + ""); } else if (diversity >= 0.8 && diversity < 0.9) { int count = map.get("0.8-0.9") != null ? Integer.parseInt(map.get("0.8-0.9")) + 1 : 1; map.put("0.8-0.9", count + ""); } else if (diversity >= 0.9) { int count = map.get("0.9-1") != null ? Integer.parseInt(map.get("0.9-1")) + 1 : 1; map.put("0.9-1", count + ""); } } JSONObject json = new JSONObject(map); writeFile(prop.getProperty("metrics_file"), json.toJSONString()); return new IRStatisticsImplCustom(precision.getAverage(), recall.getAverage(), fallOut.getAverage(), nDCG.getAverage(), (double) numUsersWithRecommendations / (double) numUsersRecommendedFor); }