List of usage examples for org.apache.commons.lang.time DateFormatUtils format
public static String format(Date date, String pattern)
Formats a date/time into a specific pattern.
From source file:org.dspace.statistics.ElasticSearchLogger.java
public void post(DSpaceObject dspaceObject, HttpServletRequest request, EPerson currentUser) { //log.info("DS-ES post for type:"+dspaceObject.getType() + " -- " + dspaceObject.getName()); client = ElasticSearchLogger.getInstance().getClient(); boolean isSpiderBot = SpiderDetector.isSpider(request); try {/*from w w w.j av a2 s . co m*/ if (isSpiderBot && !ConfigurationManager.getBooleanProperty("usage-statistics", "logBots", true)) { return; } // Save our basic info that we already have String ip = request.getRemoteAddr(); if (isUseProxies() && request.getHeader("X-Forwarded-For") != null) { /* This header is a comma delimited list */ for (String xfip : request.getHeader("X-Forwarded-For").split(",")) { /* proxy itself will sometime populate this header with the same value in remote address. ordering in spec is vague, we'll just take the last not equal to the proxy */ if (!request.getHeader("X-Forwarded-For").contains(ip)) { ip = xfip.trim(); } } } XContentBuilder docBuilder = null; docBuilder = XContentFactory.jsonBuilder().startObject(); docBuilder.field("ip", ip); docBuilder.field("id", dspaceObject.getID()); // The numerical constant that represents the DSpaceObject TYPE. i.e. 0=bitstream, 2=item, ... docBuilder.field("typeIndex", dspaceObject.getType()); // The text that represent the DSpaceObject TYPE. i.e. BITSTREAM, ITEM, COLLECTION, COMMUNITY docBuilder.field("type", Constants.typeText[dspaceObject.getType()]); // Save the current time docBuilder.field("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601)); if (currentUser != null) { docBuilder.field("epersonid", currentUser.getID()); } try { String dns = DnsLookup.reverseDns(ip); docBuilder.field("dns", dns.toLowerCase()); } catch (Exception e) { log.error("Failed DNS Lookup for IP:" + ip); log.debug(e.getMessage(), e); } // Save the location information if valid, save the event without // location information if not valid Location location = locationService.getLocation(ip); if (location != null && !("--".equals(location.countryCode) && location.latitude == -180 && location.longitude == -180)) { try { docBuilder.field("continent", LocationUtils.getContinentCode(location.countryCode)); } catch (Exception e) { System.out.println("COUNTRY ERROR: " + location.countryCode); } docBuilder.field("countryCode", location.countryCode); docBuilder.field("city", location.city); docBuilder.field("latitude", location.latitude); docBuilder.field("longitude", location.longitude); docBuilder.field("isBot", isSpiderBot); if (request.getHeader("User-Agent") != null) { docBuilder.field("userAgent", request.getHeader("User-Agent")); } } if (dspaceObject instanceof Bitstream) { Bitstream bit = (Bitstream) dspaceObject; Bundle[] bundles = bit.getBundles(); docBuilder.field("bundleName").startArray(); for (Bundle bundle : bundles) { docBuilder.value(bundle.getName()); } docBuilder.endArray(); } storeParents(docBuilder, getParents(dspaceObject)); docBuilder.endObject(); if (docBuilder != null) { IndexRequestBuilder irb = client.prepareIndex(indexName, indexType).setSource(docBuilder); //log.info("Executing document insert into index"); if (client == null) { log.error("Hey, client is null"); } irb.execute().actionGet(); } } catch (RuntimeException re) { log.error("RunTimer in ESL:\n" + ExceptionUtils.getStackTrace(re)); throw re; } catch (Exception e) { log.error(e.getMessage()); } finally { client.close(); } }
From source file:org.dspace.statistics.ElasticSearchLoggerServiceImpl.java
@Override public void post(DSpaceObject dspaceObject, HttpServletRequest request, EPerson currentUser) { //log.info("DS-ES post for type:"+dspaceObject.getType() + " -- " + dspaceObject.getName()); client = getClient();/*from w ww . ja v a 2 s .c om*/ boolean isSpiderBot = SpiderDetector.isSpider(request); try { if (isSpiderBot && !ConfigurationManager.getBooleanProperty("usage-statistics", "logBots", true)) { return; } // Save our basic info that we already have String ip = request.getRemoteAddr(); if (isUseProxies() && request.getHeader("X-Forwarded-For") != null) { /* This header is a comma delimited list */ for (String xfip : request.getHeader("X-Forwarded-For").split(",")) { /* proxy itself will sometime populate this header with the same value in remote address. ordering in spec is vague, we'll just take the last not equal to the proxy */ if (!request.getHeader("X-Forwarded-For").contains(ip)) { ip = xfip.trim(); } } } if (!isUseProxies() && request.getHeader("X-Forwarded-For") != null) { log.warn( "X-Forwarded-For header detected but useProxies is not enabled. If your dspace is behind a proxy set it to true"); } XContentBuilder docBuilder = null; docBuilder = XContentFactory.jsonBuilder().startObject(); docBuilder.field("ip", ip); docBuilder.field("id", dspaceObject.getID()); // The numerical constant that represents the DSpaceObject TYPE. i.e. 0=bitstream, 2=item, ... docBuilder.field("typeIndex", dspaceObject.getType()); // The text that represent the DSpaceObject TYPE. i.e. BITSTREAM, ITEM, COLLECTION, COMMUNITY docBuilder.field("type", Constants.typeText[dspaceObject.getType()]); // Save the current time docBuilder.field("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601)); if (currentUser != null) { docBuilder.field("epersonid", currentUser.getID()); } try { String dns = DnsLookup.reverseDns(ip); docBuilder.field("dns", dns.toLowerCase()); } catch (Exception e) { log.info("Failed DNS Lookup for IP:" + ip); log.debug(e.getMessage(), e); } // Save the location information if valid, save the event without // location information if not valid Location location = locationService.getLocation(ip); if (location != null && !("--".equals(location.countryCode) && location.latitude == -180 && location.longitude == -180)) { try { docBuilder.field("continent", LocationUtils.getContinentCode(location.countryCode)); } catch (Exception e) { System.out.println("COUNTRY ERROR: " + location.countryCode); } docBuilder.field("countryCode", location.countryCode); docBuilder.field("city", location.city); docBuilder.field("latitude", location.latitude); docBuilder.field("longitude", location.longitude); docBuilder.field("isBot", isSpiderBot); if (request.getHeader("User-Agent") != null) { docBuilder.field("userAgent", request.getHeader("User-Agent")); } } if (dspaceObject instanceof Bitstream) { Bitstream bit = (Bitstream) dspaceObject; List<Bundle> bundles = bit.getBundles(); docBuilder.field("bundleName").startArray(); for (Bundle bundle : bundles) { docBuilder.value(bundle.getName()); } docBuilder.endArray(); } storeParents(docBuilder, getParents(dspaceObject)); docBuilder.endObject(); if (docBuilder != null) { IndexRequestBuilder irb = client.prepareIndex(indexName, indexType).setSource(docBuilder); //log.info("Executing document insert into index"); if (client == null) { log.error("Hey, client is null"); } irb.execute().actionGet(); } } catch (RuntimeException re) { log.error("RunTimer in ESL:\n" + ExceptionUtils.getStackTrace(re)); throw re; } catch (Exception e) { log.error(e.getMessage()); } finally { client.close(); } }
From source file:org.dspace.statistics.ElasticSearchLoggerServiceImpl.java
@Override public void post(DSpaceObject dspaceObject, String ip, String userAgent, String xforwardedfor, EPerson currentUser) {/* w w w . ja v a 2s . c om*/ //log.info("DS-ES post for type:"+dspaceObject.getType() + " -- " + dspaceObject.getName()); client = getClient(); boolean isSpiderBot = SpiderDetector.isSpider(ip); try { if (isSpiderBot && !ConfigurationManager.getBooleanProperty("usage-statistics", "logBots", true)) { return; } // Save our basic info that we already have if (isUseProxies() && xforwardedfor != null) { /* This header is a comma delimited list */ for (String xfip : xforwardedfor.split(",")) { /* proxy itself will sometime populate this header with the same value in remote address. ordering in spec is vague, we'll just take the last not equal to the proxy */ if (!xforwardedfor.contains(ip)) { ip = xfip.trim(); } } } if (!isUseProxies() && xforwardedfor != null) { log.warn( "X-Forwarded-For header detected but useProxies is not enabled. If your dspace is behind a proxy set it to true"); } XContentBuilder docBuilder = null; docBuilder = XContentFactory.jsonBuilder().startObject(); docBuilder.field("ip", ip); docBuilder.field("id", dspaceObject.getID()); // The numerical constant that represents the DSpaceObject TYPE. i.e. 0=bitstream, 2=item, ... docBuilder.field("typeIndex", dspaceObject.getType()); // The text that represent the DSpaceObject TYPE. i.e. BITSTREAM, ITEM, COLLECTION, COMMUNITY docBuilder.field("type", Constants.typeText[dspaceObject.getType()]); // Save the current time docBuilder.field("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601)); if (currentUser != null) { docBuilder.field("epersonid", currentUser.getID()); } try { String dns = DnsLookup.reverseDns(ip); docBuilder.field("dns", dns.toLowerCase()); } catch (Exception e) { log.info("Failed DNS Lookup for IP:" + ip); log.debug(e.getMessage(), e); } // Save the location information if valid, save the event without // location information if not valid Location location = locationService.getLocation(ip); if (location != null && !("--".equals(location.countryCode) && location.latitude == -180 && location.longitude == -180)) { try { docBuilder.field("continent", LocationUtils.getContinentCode(location.countryCode)); } catch (Exception e) { System.out.println("COUNTRY ERROR: " + location.countryCode); } docBuilder.field("countryCode", location.countryCode); docBuilder.field("city", location.city); docBuilder.field("latitude", location.latitude); docBuilder.field("longitude", location.longitude); docBuilder.field("isBot", isSpiderBot); if (userAgent != null) { docBuilder.field("userAgent", userAgent); } } if (dspaceObject instanceof Bitstream) { Bitstream bit = (Bitstream) dspaceObject; List<Bundle> bundles = bit.getBundles(); docBuilder.field("bundleName").startArray(); for (Bundle bundle : bundles) { docBuilder.value(bundle.getName()); } docBuilder.endArray(); } storeParents(docBuilder, getParents(dspaceObject)); docBuilder.endObject(); if (docBuilder != null) { IndexRequestBuilder irb = client.prepareIndex(indexName, indexType).setSource(docBuilder); //log.info("Executing document insert into index"); if (client == null) { log.error("Hey, client is null"); } irb.execute().actionGet(); } } catch (RuntimeException re) { log.error("RunTimer in ESL:\n" + ExceptionUtils.getStackTrace(re)); throw re; } catch (Exception e) { log.error(e.getMessage()); } finally { client.close(); } }
From source file:org.dspace.statistics.SolrLogger.java
/** * Returns a solr input document containing common information about the statistics * regardless if we are logging a search or a view of a DSpace object * @param dspaceObject the object used.//from www.j av a2 s .c o m * @param request the current request context. * @param currentUser the current session's user. * @return a solr input document * @throws SQLException in case of a database exception */ private static SolrInputDocument getCommonSolrDoc(DSpaceObject dspaceObject, HttpServletRequest request, EPerson currentUser) throws SQLException { boolean isSpiderBot = request != null && SpiderDetector.isSpider(request); if (isSpiderBot && !ConfigurationManager.getBooleanProperty("usage-statistics", "logBots", true)) { return null; } SolrInputDocument doc1 = new SolrInputDocument(); // Save our basic info that we already have if (request != null) { String ip = request.getRemoteAddr(); if (isUseProxies() && request.getHeader("X-Forwarded-For") != null) { /* This header is a comma delimited list */ for (String xfip : request.getHeader("X-Forwarded-For").split(",")) { /* proxy itself will sometime populate this header with the same value in remote address. ordering in spec is vague, we'll just take the last not equal to the proxy */ if (!request.getHeader("X-Forwarded-For").contains(ip)) { ip = xfip.trim(); } } } doc1.addField("ip", ip); //Also store the referrer if (request.getHeader("referer") != null) { doc1.addField("referrer", request.getHeader("referer")); } try { String dns = DnsLookup.reverseDns(ip); doc1.addField("dns", dns.toLowerCase()); } catch (Exception e) { log.error("Failed DNS Lookup for IP:" + ip); log.debug(e.getMessage(), e); } // Save the location information if valid, save the event without // location information if not valid if (locationService != null) { Location location = locationService.getLocation(ip); if (location != null && !("--".equals(location.countryCode) && location.latitude == -180 && location.longitude == -180)) { try { doc1.addField("continent", LocationUtils.getContinentCode(location.countryCode)); } catch (Exception e) { System.out.println("COUNTRY ERROR: " + location.countryCode); } doc1.addField("countryCode", location.countryCode); doc1.addField("city", location.city); doc1.addField("latitude", location.latitude); doc1.addField("longitude", location.longitude); doc1.addField("isBot", isSpiderBot); if (request.getHeader("User-Agent") != null) { doc1.addField("userAgent", request.getHeader("User-Agent")); } } } } if (dspaceObject != null) { doc1.addField("id", dspaceObject.getID()); doc1.addField("type", dspaceObject.getType()); storeParents(doc1, dspaceObject); } // Save the current time doc1.addField("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601)); if (currentUser != null) { doc1.addField("epersonid", currentUser.getID()); } return doc1; }
From source file:org.dspace.statistics.SolrLoggerServiceImpl.java
/** * Returns a solr input document containing common information about the statistics * regardless if we are logging a search or a view of a DSpace object * @param dspaceObject the object used.//w w w . j a v a2 s . c om * @param request the current request context. * @param currentUser the current session's user. * @return a solr input document * @throws SQLException in case of a database exception */ protected SolrInputDocument getCommonSolrDoc(DSpaceObject dspaceObject, HttpServletRequest request, EPerson currentUser) throws SQLException { boolean isSpiderBot = request != null && SpiderDetector.isSpider(request); if (isSpiderBot && !configurationService.getBooleanProperty("usage-statistics.logBots", true)) { return null; } SolrInputDocument doc1 = new SolrInputDocument(); // Save our basic info that we already have if (request != null) { String ip = request.getRemoteAddr(); if (isUseProxies() && request.getHeader("X-Forwarded-For") != null) { /* This header is a comma delimited list */ for (String xfip : request.getHeader("X-Forwarded-For").split(",")) { /* proxy itself will sometime populate this header with the same value in remote address. ordering in spec is vague, we'll just take the last not equal to the proxy */ if (!request.getHeader("X-Forwarded-For").contains(ip)) { ip = xfip.trim(); } } } if (!isUseProxies() && request.getHeader("X-Forwarded-For") != null) { log.warn( "X-Forwarded-For header detected but useProxies is not enabled. If your dspace is behind a proxy set it to true"); } doc1.addField("ip", ip); //Also store the referrer if (request.getHeader("referer") != null) { doc1.addField("referrer", request.getHeader("referer")); } try { String dns = DnsLookup.reverseDns(ip); doc1.addField("dns", dns.toLowerCase()); } catch (Exception e) { log.info("Failed DNS Lookup for IP:" + ip); log.debug(e.getMessage(), e); } if (request.getHeader("User-Agent") != null) { doc1.addField("userAgent", request.getHeader("User-Agent")); } doc1.addField("isBot", isSpiderBot); // Save the location information if valid, save the event without // location information if not valid if (locationService != null) { Location location = locationService.getLocation(ip); if (location != null && !("--".equals(location.countryCode) && location.latitude == -180 && location.longitude == -180)) { try { doc1.addField("continent", LocationUtils.getContinentCode(location.countryCode)); } catch (Exception e) { System.out.println("COUNTRY ERROR: " + location.countryCode); } doc1.addField("countryCode", location.countryCode); doc1.addField("city", location.city); doc1.addField("latitude", location.latitude); doc1.addField("longitude", location.longitude); } } } if (dspaceObject != null) { doc1.addField("id", dspaceObject.getID()); doc1.addField("type", dspaceObject.getType()); storeParents(doc1, dspaceObject); } // Save the current time doc1.addField("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601)); if (currentUser != null) { doc1.addField("epersonid", currentUser.getID()); } return doc1; }
From source file:org.dspace.statistics.SolrLoggerServiceImpl.java
protected SolrInputDocument getCommonSolrDoc(DSpaceObject dspaceObject, String ip, String userAgent, String xforwardedfor, EPerson currentUser) throws SQLException { boolean isSpiderBot = SpiderDetector.isSpider(ip); if (isSpiderBot && !configurationService.getBooleanProperty("usage-statistics.logBots", true)) { return null; }//from w w w.ja v a2 s.co m SolrInputDocument doc1 = new SolrInputDocument(); // Save our basic info that we already have if (!isUseProxies() && xforwardedfor != null) { log.warn( "X-Forwarded-For header detected but useProxies is not enabled. If your dspace is behind a proxy set it to true"); } if (isUseProxies() && xforwardedfor != null) { /* This header is a comma delimited list */ for (String xfip : xforwardedfor.split(",")) { /* proxy itself will sometime populate this header with the same value in remote address. ordering in spec is vague, we'll just take the last not equal to the proxy */ if (!xforwardedfor.contains(ip)) { ip = xfip.trim(); } } doc1.addField("ip", ip); try { String dns = DnsLookup.reverseDns(ip); doc1.addField("dns", dns.toLowerCase()); } catch (Exception e) { log.info("Failed DNS Lookup for IP:" + ip); log.debug(e.getMessage(), e); } if (userAgent != null) { doc1.addField("userAgent", userAgent); } doc1.addField("isBot", isSpiderBot); // Save the location information if valid, save the event without // location information if not valid if (locationService != null) { Location location = locationService.getLocation(ip); if (location != null && !("--".equals(location.countryCode) && location.latitude == -180 && location.longitude == -180)) { try { doc1.addField("continent", LocationUtils.getContinentCode(location.countryCode)); } catch (Exception e) { System.out.println("COUNTRY ERROR: " + location.countryCode); } doc1.addField("countryCode", location.countryCode); doc1.addField("city", location.city); doc1.addField("latitude", location.latitude); doc1.addField("longitude", location.longitude); } } } if (dspaceObject != null) { doc1.addField("id", dspaceObject.getID()); doc1.addField("type", dspaceObject.getType()); storeParents(doc1, dspaceObject); } // Save the current time doc1.addField("time", DateFormatUtils.format(new Date(), DATE_FORMAT_8601)); if (currentUser != null) { doc1.addField("epersonid", currentUser.getID()); } return doc1; }
From source file:org.dspace.statistics.util.StatisticsDataGenerator.java
public static void main(String[] args) throws Exception { CommandLineParser parser = new PosixParser(); Options options = new Options(); options.addOption("n", "nrlogs", true, "type: nr of logs to be generated"); options.addOption("s", "startDate", true, "type: the start date from which we start generating our logs"); options.addOption("e", "endDate", true, "type: the end date from which we start generating our logs"); options.addOption("a", "cms", true, "The starting id of our community"); options.addOption("b", "cme", true, "The end id of our community"); options.addOption("c", "cls", true, "The starting id of our collection"); options.addOption("d", "cle", true, "The end if of our collection"); options.addOption("f", "is", true, "The starting id of our item"); options.addOption("g", "ie", true, "The end id of our item"); options.addOption("h", "bs", true, "The starting id of our bitstream"); options.addOption("i", "be", true, "The end id of our bitstream"); options.addOption("j", "ps", true, "The starting id of our epersons"); options.addOption("k", "pe", true, "The end id of our epersons"); CommandLine line = parser.parse(options, args); int nrLogs;/*from w w w. jav a2s. c o m*/ long startDate; long endDate; long commStartId; long commEndId; long collStartId; long collEndId; long itemStartId; long itemEndId; long bitStartId; long bitEndId; long epersonStartId; long epersonEndId; if (line.hasOption("n")) { nrLogs = Integer.parseInt(line.getOptionValue("n")); } else { System.out.println("We need to know how many logs we need to create"); return; } if (line.hasOption("s")) { startDate = getDateInMiliseconds(line.getOptionValue("s")); } else { startDate = getDateInMiliseconds("01/01/2006"); } if (line.hasOption("e")) { endDate = getDateInMiliseconds(line.getOptionValue("e")); } else { endDate = new Date().getTime(); } if (line.hasOption("a")) { commStartId = Long.parseLong(line.getOptionValue("a")); } else { return; } if (line.hasOption("b")) { commEndId = Long.parseLong(line.getOptionValue("b")); } else { return; } if (line.hasOption("c")) { collStartId = Long.parseLong(line.getOptionValue("c")); } else { return; } if (line.hasOption("d")) { collEndId = Long.parseLong(line.getOptionValue("d")); } else { return; } if (line.hasOption("f")) { itemStartId = Long.parseLong(line.getOptionValue("f")); } else { return; } if (line.hasOption("g")) { itemEndId = Long.parseLong(line.getOptionValue("g")); } else { return; } if (line.hasOption("h")) { bitStartId = Long.parseLong(line.getOptionValue("h")); } else { return; } if (line.hasOption("i")) { bitEndId = Long.parseLong(line.getOptionValue("i")); } else { return; } if (line.hasOption("j")) { epersonStartId = Long.parseLong(line.getOptionValue("j")); } else { return; } if (line.hasOption("k")) { epersonEndId = Long.parseLong(line.getOptionValue("k")); } else { return; } // Get the max id range long maxIdTotal = Math.max(commEndId, collEndId); maxIdTotal = Math.max(maxIdTotal, itemEndId); maxIdTotal = Math.max(maxIdTotal, bitEndId); // We got 3/4 chance the person visting the dso is not logged in epersonEndId *= 4; // We got all our parameters now get the rest Context context = new Context(); // Find our solr server CommonsHttpSolrServer solr = new CommonsHttpSolrServer( ConfigurationManager.getProperty("solr-statistics", "server")); solr.deleteByQuery("*:*"); solr.commit(); Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo(); String prevIp = null; String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); LookupService cl = new LookupService(dbfile, LookupService.GEOIP_STANDARD); int countryErrors = 0; for (int i = 0; i < nrLogs; i++) { String ip = ""; Date time; String continent; String countryCode; float longitude; float latitude; String city; // 1. Generate an ip for our user StringBuilder ipBuilder = new StringBuilder(); for (int j = 0; j < 4; j++) { ipBuilder.append(getRandomNumberInRange(0, 254)); if (j != 3) { ipBuilder.append("."); } } ip = ipBuilder.toString(); // 2 Depending on our ip get all the location info Location location; try { location = cl.getLocation(ip); } catch (Exception e) { location = null; } if (location == null) { // If we haven't got a prev ip this is pretty useless so move on // to the next one if (prevIp == null) { continue; } ip = prevIp; location = cl.getLocation(ip); } city = location.city; countryCode = location.countryCode; longitude = location.longitude; latitude = location.latitude; try { continent = LocationUtils.getContinentCode(countryCode); } catch (Exception e) { // We could get an error if our country == Europa this doesn't // matter for generating statistics so ignore it System.out.println("COUNTRY ERROR: " + countryCode); countryErrors++; continue; } // 3. Generate a date that the object was visited time = new Date(getRandomNumberInRange(startDate, endDate)); // 4. Get our dspaceobject we are supposed to be working on // We got mostly item views so lets say we got 1/2 chance that we // got an item view // What type have we got (PS: I know we haven't got 5 as a dso type // we can log but it is used so our item gets move traffic) int type = (int) getRandomNumberInRange(0, 8); if (type == Constants.BUNDLE || type >= 5) { type = Constants.ITEM; } int dsoId = -1; // Now we need to find a valid id switch (type) { case Constants.COMMUNITY: dsoId = (int) getRandomNumberInRange(commStartId, commEndId); break; case Constants.COLLECTION: dsoId = (int) getRandomNumberInRange(collStartId, collEndId); break; case Constants.ITEM: dsoId = (int) getRandomNumberInRange(itemStartId, itemEndId); break; case Constants.BITSTREAM: dsoId = (int) getRandomNumberInRange(bitStartId, bitEndId); break; } // Now find our dso DSpaceObject dso = DSpaceObject.find(context, type, dsoId); if (dso instanceof Bitstream) { Bitstream bit = (Bitstream) dso; if (bit.getFormat().isInternal()) { dso = null; } } // Make sure we got a dso boolean substract = false; while (dso == null) { // If our dsoId gets higher then our maxIdtotal we need to lower // to find a valid id if (dsoId == maxIdTotal) { substract = true; } if (substract) { dsoId--; } else { dsoId++; } dso = DSpaceObject.find(context, type, dsoId); if (dso instanceof Bitstream) { Bitstream bit = (Bitstream) dso; if (bit.getFormat().isInternal()) { dso = null; } } // System.out.println("REFIND"); } // Find the person who is visting us int epersonId = (int) getRandomNumberInRange(epersonStartId, epersonEndId); EPerson eperson = EPerson.find(context, epersonId); if (eperson == null) { epersonId = -1; } // System.out.println(ip); // System.out.println(country + " " + // LocationUtils.getCountryName(countryCode)); // Resolve the dns String dns = null; try { dns = DnsLookup.reverseDns(ip); } catch (Exception e) { } System.out.println(ip); System.out.println(dns); // Save it in our server SolrInputDocument doc1 = new SolrInputDocument(); doc1.addField("ip", ip); doc1.addField("type", dso.getType()); doc1.addField("id", dso.getID()); doc1.addField("time", DateFormatUtils.format(time, SolrLogger.DATE_FORMAT_8601)); doc1.addField("continent", continent); // doc1.addField("country", country); doc1.addField("countryCode", countryCode); doc1.addField("city", city); doc1.addField("latitude", latitude); doc1.addField("longitude", longitude); if (epersonId > 0) { doc1.addField("epersonid", epersonId); } if (dns != null) { doc1.addField("dns", dns.toLowerCase()); } if (dso instanceof Item) { Item item = (Item) dso; // Store the metadata for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet()) { String dcField = entry.getValue(); DCValue[] vals = item.getMetadata(dcField.split("\\.")[0], dcField.split("\\.")[1], dcField.split("\\.")[2], Item.ANY); for (DCValue val1 : vals) { String val = val1.value; doc1.addField(entry.getKey(), val); doc1.addField(entry.getKey() + "_search", val.toLowerCase()); } } } SolrLogger.storeParents(doc1, dso); solr.add(doc1); // Make sure we have a previous ip prevIp = ip; } System.out.println("Nr of countryErrors: " + countryErrors); // Commit at the end cause it takes a while solr.commit(); }
From source file:org.dspace.statistics.util.StatisticsImporter.java
/** * Read lines from the statistics file and load their data into solr. * * @param filename The filename of the file to load * @param context The DSpace Context/* ww w. j av a 2 s . com*/ * @param verbose Whether to display verbose output */ private void load(String filename, Context context, boolean verbose) { // Item counter int counter = 0; int errors = 0; int searchengines = 0; try { BufferedReader input; if (null == filename || "-".equals(filename)) { input = new BufferedReader(new InputStreamReader(System.in)); filename = "standard input"; } else input = new BufferedReader(new FileReader(new File(filename))); // Print out the filename for confirmation System.out.println("Processing file: " + filename); String line; // String uuid; String action; String id; Date date; String user; String ip; String continent = ""; String country = ""; String countryCode = ""; float longitude = 0f; float latitude = 0f; String city = ""; String dns; DNSCache dnsCache = new DNSCache(2500, 0.75f, 2500); Object fromCache; Random rand = new Random(); while ((line = input.readLine()) != null) { // Tokenise the line String data = ""; counter++; errors++; if (verbose) { System.out.println("Line:" + line); } String[] parts = line.split(","); // uuid = parts[0]; action = parts[1]; id = parts[2]; date = dateFormat.parse(parts[3]); user = parts[4]; ip = parts[5]; // Resolve the dns (if applicable) to get rid of search engine bots early on in the processing chain dns = ""; if (!skipReverseDNS) { // Is the IP address in the cache? fromCache = dnsCache.get(ip); if (fromCache != null) { dns = (String) fromCache; } else { try { dns = DnsLookup.reverseDns(ip); dnsCache.put(ip, dns); } catch (Exception e) { dns = ""; } } } data += ("ip addr = " + ip); data += (", dns name = " + dns); if ((dns.endsWith(".googlebot.com.")) || (dns.endsWith(".crawl.yahoo.net.")) || (dns.endsWith(".search.msn.com."))) { if (verbose) { System.out.println(data + ", IGNORE (search engine)"); } errors--; searchengines++; continue; } // Get the geo information for the user Location location; try { location = geoipLookup.getLocation(ip); city = location.city; country = location.countryName; countryCode = location.countryCode; longitude = location.longitude; latitude = location.latitude; if (verbose) { data += (", country = " + country); data += (", city = " + city); System.out.println(data); } try { continent = LocationUtils.getContinentCode(countryCode); } catch (Exception e) { if (verbose) { System.out.println("Unknown country code: " + countryCode); } continue; } } catch (Exception e) { // No problem - just can't look them up } // Now find our dso int type = 0; if ("view_bitstream".equals(action)) { type = Constants.BITSTREAM; if (useLocal) { id = "" + localBitstreams.get(rand.nextInt(localBitstreams.size())); } } else if ("view_item".equals(action)) { type = Constants.ITEM; if (useLocal) { id = "" + localItems.get(rand.nextInt(localItems.size())); } } else if ("view_collection".equals(action)) { type = Constants.COLLECTION; if (useLocal) { id = "" + localCollections.get(rand.nextInt(localCollections.size())); } } else if ("view_community".equals(action)) { type = Constants.COMMUNITY; if (useLocal) { id = "" + localCommunities.get(rand.nextInt(localCommunities.size())); } } DSpaceObject dso = DSpaceObject.find(context, type, Integer.parseInt(id)); if (dso == null) { if (verbose) { System.err.println(" - DSO with ID '" + id + "' is no longer in the system"); } continue; } // Get the eperson details EPerson eperson = EPerson.findByEmail(context, user); int epersonId = 0; if (eperson != null) { eperson.getID(); } // Save it in our server SolrInputDocument sid = new SolrInputDocument(); sid.addField("ip", ip); sid.addField("type", dso.getType()); sid.addField("id", dso.getID()); sid.addField("time", DateFormatUtils.format(date, SolrLogger.DATE_FORMAT_8601)); sid.addField("continent", continent); sid.addField("country", country); sid.addField("countryCode", countryCode); sid.addField("city", city); sid.addField("latitude", latitude); sid.addField("longitude", longitude); if (epersonId > 0) { sid.addField("epersonid", epersonId); } if (dns != null) { sid.addField("dns", dns.toLowerCase()); } if (dso instanceof Item) { Item item = (Item) dso; // Store the metadata for (String storedField : metadataStorageInfo.keySet()) { String dcField = metadataStorageInfo.get(storedField); DCValue[] vals = item.getMetadata(dcField.split("\\.")[0], dcField.split("\\.")[1], dcField.split("\\.")[2], Item.ANY); for (DCValue val1 : vals) { String val = val1.value; sid.addField(String.valueOf(storedField), val); sid.addField(String.valueOf(storedField + "_search"), val.toLowerCase()); } } } SolrLogger.storeParents(sid, dso); solr.add(sid); errors--; } } catch (RuntimeException re) { throw re; } catch (Exception e) { System.err.println(e.getMessage()); log.error(e.getMessage(), e); } DecimalFormat percentage = new DecimalFormat("##.###"); int committed = counter - errors - searchengines; System.out.println("Processed " + counter + " log lines"); if (counter > 0) { Double committedpercentage = 100d * committed / counter; System.out.println( " - " + committed + " entries added to solr: " + percentage.format(committedpercentage) + "%"); Double errorpercentage = 100d * errors / counter; System.out.println(" - " + errors + " errors: " + percentage.format(errorpercentage) + "%"); Double sepercentage = 100d * searchengines / counter; System.out.println(" - " + searchengines + " search engine activity skipped: " + percentage.format(sepercentage) + "%"); System.out.print("About to commit data to solr..."); // Commit at the end because it takes a while try { solr.commit(); } catch (SolrServerException sse) { System.err.println("Error committing statistics to solr server!"); sse.printStackTrace(); System.exit(1); } catch (IOException ioe) { System.err.println("Error writing to solr server!"); ioe.printStackTrace(); System.exit(1); } } System.out.println(" done!"); }
From source file:org.dspace.statistics.util.StatisticsImporterElasticSearch.java
/** * Read lines from the statistics file and load their data into Elastic Search. * * @param filename The filename of the file to load * @param context The DSpace Context//from w w w . ja va 2 s . c o m * @param verbose Whether to display verbose output */ private void load(String filename, Context context, boolean verbose) { // Item counter int counter = 0; int errors = 0; int searchengines = 0; try { BufferedReader input; if (null == filename || "-".equals(filename)) { input = new BufferedReader(new InputStreamReader(System.in)); filename = "standard input"; } else input = new BufferedReader(new FileReader(new File(filename))); // Print out the filename for confirmation System.out.println("Processing file: " + filename); String line; // String uuid; String action; String id; Date date; String user; String ip; String continent = ""; String country = ""; String countryCode = ""; float longitude = 0f; float latitude = 0f; String city = ""; String dns; DNSCache dnsCache = new DNSCache(2500, 0.75f, 2500); Object fromCache; Random rand = new Random(); while ((line = input.readLine()) != null) { // Tokenise the line counter++; errors++; if (verbose) { System.out.println("Line:" + line); } String[] parts = line.split(","); // uuid = parts[0]; action = parts[1]; id = parts[2]; date = dateFormat.parse(parts[3]); user = parts[4]; ip = parts[5]; // Resolve the dns (if applicable) to get rid of search engine bots early on in the processing chain dns = ""; if (!skipReverseDNS) { // Is the IP address in the cache? fromCache = dnsCache.get(ip); if (fromCache != null) { dns = (String) fromCache; } else { try { dns = DnsLookup.reverseDns(ip); dnsCache.put(ip, dns); } catch (Exception e) { dns = ""; } } } String data = ""; data += ("ip addr = " + ip); data += (", dns name = " + dns); if ((dns.endsWith(".googlebot.com.")) || (dns.endsWith(".crawl.yahoo.net.")) || (dns.endsWith(".search.msn.com."))) { if (verbose) { System.out.println(data + ", IGNORE (search engine)"); } errors--; searchengines++; continue; } // Get the geo information for the user Location location; try { location = geoipLookup.getLocation(ip); city = location.city; country = location.countryName; countryCode = location.countryCode; longitude = location.longitude; latitude = location.latitude; if (verbose) { data += (", country = " + country); data += (", city = " + city); System.out.println(data); } try { continent = LocationUtils.getContinentCode(countryCode); } catch (Exception e) { if (verbose) { System.out.println("Unknown country code: " + countryCode); } continue; } } catch (Exception e) { // No problem - just can't look them up } // Now find our dso int type = 0; if ("view_bitstream".equals(action)) { type = Constants.BITSTREAM; } else if ("view_item".equals(action)) { type = Constants.ITEM; } else if ("view_collection".equals(action)) { type = Constants.COLLECTION; } else if ("view_community".equals(action)) { type = Constants.COMMUNITY; } DSpaceObject dso = DSpaceObject.find(context, type, Integer.parseInt(id)); if (dso == null) { if (verbose) { System.err.println(" - DSO with ID '" + id + "' is no longer in the system"); } continue; } // Get the eperson details EPerson eperson = EPerson.findByEmail(context, user); int epersonId = 0; if (eperson != null) { eperson.getID(); } //TODO Is there any way to reuse ElasticSearchLogger.post() ? // Save it in our server XContentBuilder postBuilder = XContentFactory.jsonBuilder().startObject().field("id", dso.getID()) .field("typeIndex", dso.getType()).field("type", dso.getTypeText()) .field("geo", new GeoPoint(latitude, longitude)).field("continent", continent) .field("countryCode", countryCode).field("country", country).field("city", city) .field("ip", ip) .field("time", DateFormatUtils.format(date, SolrLogger.DATE_FORMAT_8601)); // Unable to get UserAgent from logs. .field("userAgent") if (dso instanceof Bitstream) { Bitstream bit = (Bitstream) dso; Bundle[] bundles = bit.getBundles(); postBuilder = postBuilder.field("bundleName").startArray(); for (Bundle bundle : bundles) { postBuilder = postBuilder.value(bundle.getName()); } postBuilder = postBuilder.endArray(); } if (epersonId > 0) { postBuilder = postBuilder.field("epersonid", epersonId); } if (dns != null) { postBuilder = postBuilder.field("dns", dns.toLowerCase()); } //Save for later: .field("isBot") elasticSearchLoggerInstance.storeParents(postBuilder, elasticSearchLoggerInstance.getParents(dso)); bulkRequest .add(client .prepareIndex(elasticSearchLoggerInstance.getIndexName(), elasticSearchLoggerInstance.getIndexType()) .setSource(postBuilder.endObject())); errors--; } if (bulkRequest.numberOfActions() > 0) { BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { log.error("Bulk Request Failed due to: " + bulkResponse.buildFailureMessage()); } } } catch (RuntimeException re) { throw re; } catch (Exception e) { System.err.println(e.getMessage()); log.error(e.getMessage(), e); } DecimalFormat percentage = new DecimalFormat("##.###"); int committed = counter - errors - searchengines; System.out.println("Processed " + counter + " log lines"); if (counter > 0) { Double committedpercentage = 100d * committed / counter; System.out.println(" - " + committed + " entries added to ElasticSearch: " + percentage.format(committedpercentage) + "%"); Double errorpercentage = 100d * errors / counter; System.out.println(" - " + errors + " errors: " + percentage.format(errorpercentage) + "%"); Double sepercentage = 100d * searchengines / counter; System.out.println(" - " + searchengines + " search engine activity skipped: " + percentage.format(sepercentage) + "%"); } System.out.println(" done!"); }
From source file:org.eclipse.amp.amf.testing.ide.TestRunSet.java
private String getName() { String name = ""; if (firstResource instanceof IFolder) { name += ((IFolder) firstResource).getName(); } else if (uniqueTests.size() < 3) { for (IResource res : uniqueTests) { name += res.getProjectRelativePath().removeFileExtension().lastSegment() + "_"; }//from w w w . j av a2s . c o m } else { for (IResource res : uniqueTests) { name += res.getName().substring(0, Math.max(5, res.getName().length() - 1)) + "_"; } } name += DateFormatUtils.format(System.currentTimeMillis(), "MM-dd-yy_HH-mm-S"); return name; }