Java tutorial
/* * Copyright Myrrix Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.myrrix.online.generation; import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.commons.math3.linear.RealMatrix; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import net.myrrix.common.collection.FastByIDMap; import net.myrrix.common.collection.FastIDSet; import net.myrrix.common.math.IllConditionedSolverException; import net.myrrix.common.math.MatrixUtils; import net.myrrix.common.math.Solver; import net.myrrix.online.candidate.CandidateFilter; import net.myrrix.online.candidate.CandidateFilterFactory; /** * Encapsulates one generation of the underlying recommender's model. The data in this object is quite * specific to the underlying Alternating Least Squares algorithm. A generation contains: * * <ul> * <li>X, the user-feature matrix</li> * <li>Y, the item-feature matrix</li> * <li>XLeftInverse, X's left inverse</li> * <li>YTRightInverse, the right inverse of Y transpose</li> * <li>numFeatures, the column dimension of X and Y</li> * <li>knownItemIDs, the item IDs already associated to each user</li> * <li>itemTagIDs, IDs of "users" that are really item tags</li> * <li>userTagIDs, IDs of "items" that are really user tags</li> * <li>clusters of item IDs (in distributed mode), with centroids</li> * <li>clusters of user IDs (in distributed mode), with centroids</li> * </ul> * * @author Sean Owen * @since 1.0 * @see net.myrrix.online.factorizer.MatrixFactorizer */ public final class Generation { private static final Logger log = LoggerFactory.getLogger(Generation.class); public static final String NO_KNOWN_ITEMS_KEY = "model.noKnownItems"; private final FastByIDMap<FastIDSet> knownItemIDs; private final FastByIDMap<FastIDSet> knownUserIDs; private final FastByIDMap<float[]> X; private Solver XTXsolver; private final FastByIDMap<float[]> Y; private Solver YTYsolver; private final FastIDSet itemTagIDs; private final FastIDSet userTagIDs; private final List<IDCluster> userClusters; private final List<IDCluster> itemClusters; private CandidateFilter candidateFilter; private final ReadWriteLock xLock; private final ReadWriteLock yLock; private final ReadWriteLock knownItemLock; private final ReadWriteLock knownUserLock; private final ReadWriteLock userClustersLock; private final ReadWriteLock itemClustersLock; public Generation(FastByIDMap<FastIDSet> knownItemIDs, FastByIDMap<float[]> X, FastByIDMap<float[]> Y) { this(knownItemIDs, X, Y, new FastIDSet(1000), new FastIDSet(1000)); } public Generation(FastByIDMap<FastIDSet> knownItemIDs, FastByIDMap<float[]> X, FastByIDMap<float[]> Y, FastIDSet itemTagIDs, FastIDSet userTagIDs) { this(knownItemIDs, X, Y, itemTagIDs, userTagIDs, Lists.<IDCluster>newArrayList(), Lists.<IDCluster>newArrayList()); } public Generation(FastByIDMap<FastIDSet> knownItemIDs, FastByIDMap<float[]> X, FastByIDMap<float[]> Y, FastIDSet itemTagIDs, FastIDSet userTagIDs, List<IDCluster> userClusters, List<IDCluster> itemClusters) { // knownItemIDs may be null, specially Preconditions.checkNotNull(X); Preconditions.checkNotNull(Y); Preconditions.checkNotNull(itemTagIDs); Preconditions.checkNotNull(userTagIDs); Preconditions.checkNotNull(userClusters); Preconditions.checkNotNull(itemClusters); this.knownItemIDs = knownItemIDs; this.knownUserIDs = null; // Not used yet this.X = X; this.XTXsolver = null; this.Y = Y; this.YTYsolver = null; this.itemTagIDs = itemTagIDs; this.userTagIDs = userTagIDs; this.userClusters = userClusters; this.itemClusters = itemClusters; this.candidateFilter = null; this.xLock = new ReentrantReadWriteLock(); this.yLock = new ReentrantReadWriteLock(); this.knownItemLock = new ReentrantReadWriteLock(); this.knownUserLock = null; // Not used yet this.userClustersLock = new ReentrantReadWriteLock(); this.itemClustersLock = new ReentrantReadWriteLock(); recomputeState(); } void recomputeState() { if (Boolean.parseBoolean(System.getProperty("model.solver.xtx.compute", "true"))) { XTXsolver = recomputeSolver(X, xLock.readLock()); } if (Boolean.parseBoolean(System.getProperty("model.solver.yty.compute", "true"))) { YTYsolver = recomputeSolver(Y, yLock.readLock()); } candidateFilter = CandidateFilterFactory.buildCandidateFilter(Y, yLock.readLock()); } private static Solver recomputeSolver(FastByIDMap<float[]> M, Lock readLock) { readLock.lock(); try { if (M == null || M.isEmpty()) { return null; } RealMatrix MTM = MatrixUtils.transposeTimesSelf(M); double infNorm = MTM.getNorm(); if (infNorm < 1.0) { log.warn("X'*X or Y'*Y has small inf norm ({}); try decreasing model.als.lambda", infNorm); throw new IllConditionedSolverException("infNorm: " + infNorm); } return MatrixUtils.getSolver(MTM); } finally { readLock.unlock(); } } /** * @return the number of "users" in the model (rows of {@link #getX()} */ public int getNumUsers() { return X.size(); } /** * @return the number of "items" in the model (rows of {@link #getY()} */ public int getNumItems() { return Y.size(); } /** * @return the user-feature matrix, implemented as a map from row number (user ID) to feature array */ public FastByIDMap<float[]> getX() { return X; } /** * @return {@link Solver} for the matrix X' * X */ public Solver getXTXSolver() { return XTXsolver; } /** * @return the item-feature matrix, implemented as a map from row number (item ID) to feature array */ public FastByIDMap<float[]> getY() { return Y; } /** * @return {@link Solver} for the matrix Y' * Y */ public Solver getYTYSolver() { return YTYsolver; } /** * @return the item IDs already associated to each user, as a map from user IDs to a set of item IDs */ public FastByIDMap<FastIDSet> getKnownItemIDs() { return knownItemIDs; } /** * Not used, yet. * @return null */ public FastByIDMap<FastIDSet> getKnownUserIDs() { return knownUserIDs; } /** * @return "user" IDs which are actually item tags */ public FastIDSet getItemTagIDs() { return itemTagIDs; } /** * @return "item" IDs which are actually user tags */ public FastIDSet getUserTagIDs() { return userTagIDs; } /** * @return clusters of user IDs, or {@code null} if not in distributed mode */ public List<IDCluster> getUserClusters() { return userClusters; } /** * @return clusters of item IDs, or {@code null} if not in distributed mode */ public List<IDCluster> getItemClusters() { return itemClusters; } public CandidateFilter getCandidateFilter() { return candidateFilter; } /** * Acquire this read/write lock before using {@link #getX()} or {@link #getItemTagIDs()}. */ public ReadWriteLock getXLock() { return xLock; } /** * Acquire this read/write lock before using {@link #getY()} or {@link #getUserTagIDs()}. */ public ReadWriteLock getYLock() { return yLock; } /** * Acquire this read/write lock before using {@link #getKnownItemIDs()}. */ public ReadWriteLock getKnownItemLock() { return knownItemLock; } /** * Not used, yet. * @return null */ public ReadWriteLock getKnownUserLock() { return knownUserLock; } /** * Acquire this read/write lock before using {@link #getUserClusters()}. */ public ReadWriteLock getUserClustersLock() { return userClustersLock; } /** * Acquire this read/write lock before using {@link #getItemClusters()}. */ public ReadWriteLock getItemClustersLock() { return itemClustersLock; } }