Java tutorial
package org.easyrec.utils; /**Copyright 2010 Research Studios Austria Forschungsgesellschaft mBH * * This file is part of easyrec. * * easyrec is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * easyrec is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with easyrec. If not, see <http://www.gnu.org/licenses/>. */ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.text.DateFormat; import java.util.Calendar; import java.util.Date; import java.util.Random; /** * !!! CAUTION This is a class to fill the DB with action & items * all existing DATA will be DELETED !!! * <p/> * This function creates equal distributed actions for the number of given * tenants in the given time range. * <p/> * Every created action has a random itemid, userid and actiontypeid. * <p/> * USE THIS utitlity AFTER successfully setting up the easyrec server * with your chossen operator id and password. * * @author Peter Hlavac */ public class Benchmark { private static final Log logger = LogFactory.getLog(Benchmark.class); private static final int NUMBER_OF_TENANTS = 3; private static final int NUMBER_OF_USERS = 500000; private static final int NUMBER_OF_ITEMS = 100000; private static final int NUMBER_OF_ACTIONS = 10 * 1000 * 1000 * NUMBER_OF_TENANTS; // x actions per tenant private static final int ACTION_TYPE_VIEW = 1; private static final int ACTION_TYPE_BUY = 3; private static final int ACTION_TYPE_RATE = 2; private static final int MAX_RATING_VALUE = 10; private static final int MAX_ACTIONS_PER_USER = 500; private static final int MAX_ACTIONS_ON_ITEM = 10000; private static final int ACTIONS_TIME_RANGE = 5; // this year - x years e.g. 2005-2010 private static final String OPERATOR_ID = "p"; private static final String OPERATOR_PASSWORD = "ppppp"; private static final String SIGN_IN_REQUEST = "http://localhost:8084/easyrec-web/operator/signin?operatorId=" + OPERATOR_ID + "&password=" + OPERATOR_PASSWORD; private static final String START_PLUGINS_REQUEST = "http://localhost:8084/easyrec-web/PluginStarter?operatorId=" + OPERATOR_ID + "&tenantId=EASYREC_DEMO"; private static final int[] ACTIONTYPES = { ACTION_TYPE_VIEW, ACTION_TYPE_BUY, ACTION_TYPE_RATE }; private static final String DB_HOST = "localhost"; private static final String DB_NAME = "easyrec"; private static final String DB_USER = "root"; private static final String DB_PASSWORD = "root"; private static final Random r = new Random(); private static int[] userIdDistribution = new int[NUMBER_OF_USERS]; private static int[] itemIdDistribution = new int[NUMBER_OF_ITEMS]; private static Header header = null; private static Header[] headers = null; // SQL query to decrease the number of items in action table // (works only one way, start with a high number and decrease step by step for benchmarks) // UPDATE actions SET itemid = itemdid % NEW_NUMBER_OF_ITEMS public static void main(String[] args) { Connection con = null; try { Class.forName("com.mysql.jdbc.jdbc2.optional.MysqlDataSource"); con = DriverManager.getConnection("jdbc:mysql://" + DB_HOST + "/" + DB_NAME, DB_USER, DB_PASSWORD); System.out.println("begin:" + DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.SHORT).format(new Date())); Statement st = con.createStatement(); initItemAndUserIds(); st.executeUpdate("DELETE FROM action;"); st.executeUpdate("DELETE FROM item;"); st.executeUpdate("DELETE FROM itemassoc;"); st.executeUpdate("DELETE FROM idmapping;"); System.out.println("db reset."); createItems(st); createIdMapping(st); createActions(st, NUMBER_OF_ACTIONS); startPlugins(); System.out.println("end:" + DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.SHORT).format(new Date())); } catch (Exception e) { e.printStackTrace(); } finally { if (con != null) { try { con.close(); } catch (SQLException ex) { logger.warn("An error occured", ex); } } } } private static void startPlugins() { String signin = loadWebsiteHtmlCode(SIGN_IN_REQUEST); System.out.println(signin); if (signin.contains("113")) { // TODO: Set cookie jsession ID or disable security check in RM String pluginOutput = loadWebsiteHtmlCode(START_PLUGINS_REQUEST); if (pluginOutput.contains("901")) { System.out.println("plugins finished. See plugin logs for details."); } else { System.out.println("plugins have a problem:" + pluginOutput); } } else { System.out.println("Login failed"); } } private static void createItems(Statement st) throws Exception { for (int j = 1; j <= NUMBER_OF_TENANTS; j++) { for (int i = 1; i <= NUMBER_OF_ITEMS; i++) { StringBuilder s = new StringBuilder().append(" INSERT INTO ").append(" item").append("(") .append(" tenantId,").append(" itemid,").append(" itemtype,").append(" description,") .append(" url,").append(" imageurl,").append(" active").append(") ").append("VALUE (") .append(j).append(" ,").append(i).append(" ,'ITEM',") .append(" 'The item description of item: " + i + "'") .append(" ,'http://this.is.my.tenant.com/item/url/itemid/" + i + "',") .append(" 'http://easyrec.org/img/easyrec_logo.gif?id=" + i + "',").append(" 1") .append(")"); st.executeUpdate(s.toString()); } } System.out.println("items for " + NUMBER_OF_TENANTS + " tenant(s) created."); } private static void createIdMapping(Statement st) throws Exception { for (int i = 1; i <= Math.max(NUMBER_OF_ITEMS, NUMBER_OF_USERS); i++) { StringBuilder s = new StringBuilder().append(" INSERT INTO ").append(" idmapping").append("(") .append(" intId,").append(" stringId").append(") ").append("VALUE (").append(i).append(" ,") .append(i).append(")"); st.executeUpdate(s.toString()); } System.out.println("idmapping created."); } /** * Creates actions with a random timestamp starting at the beginning of the * year until current date. * * @param st * @param actionType * @param actions * @throws Exception */ private static void createActions(Statement st, int actions) throws Exception { int i = 0; int actionTypeId = 0; Calendar actionTime = Calendar.getInstance(); Calendar currentTime = Calendar.getInstance(); while (i < actions) { i++; //actionTime.set( // actionTime.get(Calendar.YEAR), // r.nextInt(Calendar.getInstance().get(Calendar.MONTH)+1), // 1 //); actionTime.set(currentTime.get(Calendar.YEAR) - ACTIONS_TIME_RANGE + r.nextInt(ACTIONS_TIME_RANGE) + 1, r.nextInt(12) + 1, 1); actionTypeId = getActionTypeId(); StringBuilder s = new StringBuilder().append("INSERT INTO ").append(" action").append("(") .append(" tenantId,").append(" userId,").append(" sessionId,").append(" ip,") .append(" itemId,").append(" itemTypeId,").append(" actionTypeId,"); if (ACTION_TYPE_RATE == actionTypeId) s.append(" ratingValue,"); s.append(" description,").append(" actionTime").append(")").append("VALUE (") .append(r.nextInt(NUMBER_OF_TENANTS) + 1).append(" ,").append(getUserId()).append(" ,'1',") .append(" '1',").append(getItemId()).append(" ,1,").append(actionTypeId); if (ACTION_TYPE_RATE == actionTypeId) s.append(",").append(r.nextInt(MAX_RATING_VALUE)); s.append(" ,'1','").append(actionTime.get(Calendar.YEAR)).append("-") .append(actionTime.get(Calendar.MONTH) + 1).append("-") .append(r.nextInt(actionTime.getActualMaximum(Calendar.DAY_OF_MONTH)) + 1).append("'") .append(")"); st.executeUpdate(s.toString()); if (i % 100000 == 0) { System.out.println(i + " actions inserted."); } } System.out.println(i + " random actions created for " + NUMBER_OF_TENANTS + " tenant(s) with actiontypes (view: 99,4%, buy: 0,5%, rate: 0,1% "); } // on average 0.5 percent of the viewed items are bought // on average 0.1 percent of the viewed item are rated // (source: flimmit_beta, tallat) private static int getActionTypeId() { int i = r.nextInt(1000); // Random int from 0 - 999 if (i < 995) return ACTIONTYPES[0]; // view if (i < 999) return ACTIONTYPES[1]; // buy return ACTIONTYPES[2]; // rate } /** * Items are distributed as expontial occurences (longtail) * Most of the items are not interesting to the users. Only * some of them are viewed a lot. * * */ private static int getItemId() { int n = r.nextInt(itemIdDistribution[NUMBER_OF_ITEMS - 1]); for (int i = 0; i < NUMBER_OF_ITEMS; i++) { if (itemIdDistribution[i] > n) return i + 1; } return 0; } /** * Users are distributed as expontial occurences (longtail) * Most of the Users show up 1 time only very view do more actions * * */ private static int getUserId() { int n = r.nextInt(userIdDistribution[NUMBER_OF_USERS - 1]); for (int i = 0; i < NUMBER_OF_USERS; i++) { if (userIdDistribution[i] > n) return i + 1; } return 0; } private static void initItemAndUserIds() { //f(y) = 1/x mod n userIdDistribution[0] = (NUMBER_OF_USERS - 1) % MAX_ACTIONS_PER_USER; for (int i = 1; i < NUMBER_OF_USERS; i++) { userIdDistribution[i] = userIdDistribution[i - 1] + (int) (Math.round((1. / (i + 1)) * NUMBER_OF_USERS) % MAX_ACTIONS_PER_USER); } itemIdDistribution[0] = (NUMBER_OF_ITEMS - 1) % MAX_ACTIONS_ON_ITEM; for (int i = 1; i < NUMBER_OF_ITEMS; i++) { itemIdDistribution[i] = itemIdDistribution[i - 1] + (int) (Math.round((1. / (i + 1)) * NUMBER_OF_ITEMS) % MAX_ACTIONS_ON_ITEM); } System.out.println(NUMBER_OF_USERS + " userIds and " + NUMBER_OF_ITEMS + " itemIds created."); } /** * This function loads a Webpage into a string (like view source in a browser). * * @param url * */ private static String loadWebsiteHtmlCode(String url) { HttpClient httpClient = new DefaultHttpClient(); HttpGet getMethod = new HttpGet(url); String htmlCode = ""; if (header != null) { getMethod.setHeader(header); } try { HttpResponse resp = httpClient.execute(getMethod); int statusCode = resp.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { System.out.println("Method failed!" + statusCode); } // Read the response body. htmlCode = EntityUtils.toString(resp.getEntity()); header = resp.getHeaders("Set-Cookie")[0]; //TODO: read the JSESSIONID and use it in the next request. /* headers = resp.getAllHeaders(); HeaderElement[] headers = resp.getHeaders("Set-Cookie")[0].getElements(); System.out.println(resp.getHeaders("Set-Cookie")[0].getValue()); for (int i = 0; i < headers.length; i++) { HeaderElement headerElement = headers[i]; System.out.println(headerElement.getValue()); }*/ } catch (Exception e) { System.out.println(e.getMessage()); } finally { // Release the connection. //method.releaseConnection(); } return htmlCode; } }