List of usage examples for org.joda.time.format ISODateTimeFormat dateTime
public static DateTimeFormatter dateTime()
From source file:esiptestbed.mudrod.weblog.pre.SessionStatistic.java
License:Apache License
/** * Method to summarize duration, numbers of searching, viewing, and downloading requests, and * filter out suspicious sessions//from w w w . j a v a2 s . com * @throws IOException IOException * @throws InterruptedException InterruptedException * @throws ExecutionException ExecutionException */ public void processSession() throws IOException, InterruptedException, ExecutionException { es.createBulkProcessor(); String inputType = this.cleanupType; String outputType = this.sessionStats; MetricsAggregationBuilder<?> statsAgg = AggregationBuilders.stats("Stats").field("Time"); SearchResponse sr = es.getClient().prepareSearch(props.getProperty(MudrodConstants.ES_INDEX_NAME)) .setTypes(inputType).setQuery(QueryBuilders.matchAllQuery()) .addAggregation( AggregationBuilders.terms("Sessions").field("SessionID").size(0).subAggregation(statsAgg)) .execute().actionGet(); Terms Sessions = sr.getAggregations().get("Sessions"); DateTimeFormatter fmt = ISODateTimeFormat.dateTime(); String min = null; String max = null; DateTime start = null; DateTime end = null; int duration = 0; float request_rate = 0; int session_count = 0; Pattern pattern = Pattern.compile("get (.*?) http/*"); for (Terms.Bucket entry : Sessions.getBuckets()) { if (entry.getDocCount() >= 3 && !entry.getKey().equals("invalid")) { Stats agg = entry.getAggregations().get("Stats"); min = agg.getMinAsString(); max = agg.getMaxAsString(); start = fmt.parseDateTime(min); end = fmt.parseDateTime(max); duration = Seconds.secondsBetween(start, end).getSeconds(); int searchDataListRequest_count = 0; int searchDataRequest_count = 0; int searchDataListRequest_byKeywords_count = 0; int ftpRequest_count = 0; int keywords_num = 0; String IP = null; String keywords = ""; String views = ""; String downloads = ""; QueryBuilder filter_search = QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("SessionID", entry.getKey())); QueryBuilder query_search = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter_search); SearchResponse scrollResp = es.getClient().prepareSearch(props.getProperty("indexName")) .setTypes(inputType).setScroll(new TimeValue(60000)).setQuery(query_search).setSize(100) .execute().actionGet(); while (true) { for (SearchHit hit : scrollResp.getHits().getHits()) { Map<String, Object> result = hit.getSource(); String request = (String) result.get("Request"); String logType = (String) result.get("LogType"); IP = (String) result.get("IP"); Matcher matcher = pattern.matcher(request.trim().toLowerCase()); while (matcher.find()) { request = matcher.group(1); } String datasetlist = "/datasetlist?"; String dataset = "/dataset/"; if (request.contains(datasetlist)) { searchDataListRequest_count++; RequestUrl requestURL = new RequestUrl(this.props, this.es, null); String info = requestURL.getSearchInfo(request) + ","; if (!info.equals(",")) { if (keywords.equals("")) { keywords = keywords + info; } else { String[] items = info.split(","); String[] keywordList = keywords.split(","); for (int m = 0; m < items.length; m++) { if (!Arrays.asList(keywordList).contains(items[m])) { keywords = keywords + items[m] + ","; } } } } } if (request.startsWith(dataset)) { searchDataRequest_count++; if (findDataset(request) != null) { String view = findDataset(request); if ("".equals(views)) { views = view; } else { if (views.contains(view)) { } else { views = views + "," + view; } } } } if ("ftp".equals(logType)) { ftpRequest_count++; String download = ""; String requestLowercase = request.toLowerCase(); if (requestLowercase.endsWith(".jpg") == false && requestLowercase.endsWith(".pdf") == false && requestLowercase.endsWith(".txt") == false && requestLowercase.endsWith(".gif") == false) { download = request; } if ("".equals(downloads)) { downloads = download; } else { if (downloads.contains(download)) { } else { downloads = downloads + "," + download; } } } } scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (scrollResp.getHits().getHits().length == 0) { break; } } if (!keywords.equals("")) { keywords_num = keywords.split(",").length; } if (searchDataListRequest_count != 0 && searchDataListRequest_count <= Integer.parseInt(props.getProperty("searchf")) && searchDataRequest_count != 0 && searchDataRequest_count <= Integer.parseInt(props.getProperty("viewf")) && ftpRequest_count <= Integer.parseInt(props.getProperty("downloadf"))) { String sessionURL = props.getProperty("SessionPort") + props.getProperty("SessionUrl") + "?sessionid=" + entry.getKey() + "&sessionType=" + outputType + "&requestType=" + inputType; session_count++; IndexRequest ir = new IndexRequest(props.getProperty("indexName"), outputType) .source(jsonBuilder().startObject().field("SessionID", entry.getKey()) .field("SessionURL", sessionURL).field("Request_count", entry.getDocCount()) .field("Duration", duration).field("Number of Keywords", keywords_num) .field("Time", min).field("End_time", max) .field("searchDataListRequest_count", searchDataListRequest_count) .field("searchDataListRequest_byKeywords_count", searchDataListRequest_byKeywords_count) .field("searchDataRequest_count", searchDataRequest_count) .field("keywords", es.customAnalyzing(props.getProperty("indexName"), keywords)) .field("views", views).field("downloads", downloads) .field("request_rate", request_rate).field("Comments", "") .field("Validation", 0).field("Produceby", 0).field("Correlation", 0) .field("IP", IP) // .field("Coordinates", loc.latlon) .endObject()); es.getBulkProcessor().add(ir); } } } LOG.info("Session count: {}", Integer.toString(session_count)); es.destroyBulkProcessor(); }
From source file:ezbake.helpers.cdh.Cdh2EzProperties.java
License:Apache License
private String getFileComment() { return "Auto-generated by " + this.getClass().getName() + " at " + new DateTime().toString(ISODateTimeFormat.dateTime()); }
From source file:fi.hsl.parkandride.core.domain.DefaultTimeZoneDateTimeSerializer.java
License:EUPL
private static JacksonJodaFormat formatterWithTimeZone(DateTimeZone timeZone) { DateTimeFormatter formatter = ISODateTimeFormat.dateTime().withZone(timeZone); return new JacksonJodaFormat(new JacksonJodaFormat(formatter), timeZone.toTimeZone()); }
From source file:gluu.scim2.client.util.UserSerializer.java
License:MIT License
@Override public void serialize(User user, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { System.out.println(" IN UserSerializer.serialize()... "); try {/*from w w w. ja v a 2s.co m*/ jsonGenerator.writeStartObject(); ObjectMapper mapper = new ObjectMapper(); mapper.disable(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS); JsonNode rootNode = mapper.convertValue(user, JsonNode.class); Iterator<Map.Entry<String, JsonNode>> iterator = rootNode.getFields(); while (iterator.hasNext()) { Map.Entry<String, JsonNode> rootNodeEntry = iterator.next(); jsonGenerator.writeFieldName(rootNodeEntry.getKey()); if (rootNodeEntry.getKey().equals(Constants.USER_EXT_SCHEMA_ID)) { Extension extension = user.getExtension(rootNodeEntry.getKey()); Map<String, Object> list = new HashMap<String, Object>(); for (Map.Entry<String, Extension.Field> extEntry : extension.getFields().entrySet()) { if (extEntry.getValue().isMultiValued()) { if (extEntry.getValue().getType().equals(ExtensionFieldType.STRING)) { List<String> stringList = Arrays .asList(mapper.readValue(extEntry.getValue().getValue(), String[].class)); list.put(extEntry.getKey(), stringList); } else if (extEntry.getValue().getType().equals(ExtensionFieldType.DATE_TIME)) { List<Date> dateList = Arrays .asList(mapper.readValue(extEntry.getValue().getValue(), Date[].class)); List<String> stringList = new ArrayList<String>(); DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime().withZoneUTC(); for (Date date : dateList) { String dateString = dateTimeFormatter.print(date.getTime()); stringList.add(dateString); } list.put(extEntry.getKey(), stringList); } else if (extEntry.getValue().getType().equals(ExtensionFieldType.DECIMAL)) { List<BigDecimal> numberList = Arrays.asList( mapper.readValue(extEntry.getValue().getValue(), BigDecimal[].class)); list.put(extEntry.getKey(), numberList); } } else { list.put(extEntry.getKey(), extEntry.getValue().getValue()); } } jsonGenerator.writeObject(list); } else { jsonGenerator.writeObject(rootNodeEntry.getValue()); } } jsonGenerator.writeEndObject(); System.out.println(" LEAVING UserSerializer.serialize()... "); } catch (Exception e) { e.printStackTrace(); throw new IOException("Unexpected processing error; please check the input parameters."); } }
From source file:google.registry.xml.XmlTestUtils.java
License:Open Source License
/** * Deeply explore the object and normalize values so that things we consider equal compare so. * The return value consists of two parts: the updated key and the value. The value is * straightforward enough: it is the rendering of the subtree to be attached at the current point. * The key is more complicated, because of namespaces. When an XML element specifies namespaces * using xmlns attributes, those namespaces apply to the element as well as all of its * descendants. That means that, when prefixing the element name with the full namespace path, * as required to do proper comparison, the element name depends on its children. When looping * through a JSONObject map, we can't just recursively generate the value and store it using the * key. We may have to update the key as well, to get the namespaces correct. A returned key of * null indicates that we should use the existing key. A non-null key indicates that we should * replace the existing key.//from ww w. j av a 2 s.c o m * * @param elementName the name under which the current subtree was found, or null if the current * subtree's name is nonexistent or irrelevant * @param obj the current subtree * @param path the (non-namespaced) element path used for ignoredPaths purposes * @param ignoredPaths the set of paths whose values should be set to IGNORED * @param nsMap the inherited namespace identifier-to-URI map * @return the key under which the rendered subtree should be stored (or null), and the rendered * subtree */ private static Map.Entry<String, Object> normalize(@Nullable String elementName, Object obj, @Nullable String path, Set<String> ignoredPaths, Map<String, String> nsMap) throws Exception { if (obj instanceof JSONObject) { JSONObject jsonObject = (JSONObject) obj; Map<String, Object> map = new HashMap<>(); String[] names = JSONObject.getNames(jsonObject); if (names != null) { // Separate all elements and keys into namespace specifications, which we must process // first, and everything else. ImmutableList.Builder<String> namespacesBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder<String> othersBuilder = new ImmutableList.Builder<>(); for (String key : names) { (key.startsWith("xmlns") ? namespacesBuilder : othersBuilder).add(key); } // First, handle all namespace specifications, updating our ns-to-URI map. Use a HashMap // rather than an ImmutableMap.Builder so that we can override existing map entries. HashMap<String, String> newNsMap = new HashMap<>(); newNsMap.putAll(nsMap); for (String key : namespacesBuilder.build()) { // Parse the attribute name, of the form xmlns:nsid, and extract the namespace identifier. // If there's no colon, we are setting the default namespace. List<String> components = Splitter.on(':').splitToList(key); String ns = (components.size() >= 2) ? components.get(1) : ""; newNsMap.put(ns, jsonObject.get(key).toString()); } nsMap = ImmutableMap.copyOf(newNsMap); // Now, handle the non-namespace items, recursively transforming the map and mapping all // namespaces to the full URI for proper comparison. for (String key : othersBuilder.build()) { String simpleKey = Iterables.getLast(Splitter.on(':').split(key)); String newPath = (path == null) ? simpleKey : (path + "." + simpleKey); String mappedKey; Object value; if (ignoredPaths.contains(newPath)) { mappedKey = null; // Set ignored fields to a value that will compare equal. value = "IGNORED"; } else { Map.Entry<String, Object> simpleEntry = normalize(key, jsonObject.get(key), newPath, ignoredPaths, nsMap); mappedKey = simpleEntry.getKey(); value = simpleEntry.getValue(); } if (mappedKey == null) { // Note that this does not follow the XML rules exactly. I read somewhere that attribute // names, unlike element names, never use the default namespace. But after // JSONification, we cannot distinguish between attributes and child elements, so we // apply the default namespace to everything. Hopefully that will not cause a problem. mappedKey = key.equals("content") ? key : mapName(key, nsMap, true); } map.put(mappedKey, value); } } // Map the namespace of the element name of the map we are normalizing. elementName = mapName(elementName, nsMap, true); // If a node has both text content and attributes, the text content will end up under a key // called "content". If that's the only thing left (which will only happen if there was an // "xmlns:*" key that we removed), treat the node as just text and recurse. if (map.size() == 1 && map.containsKey("content")) { return new AbstractMap.SimpleEntry<>(elementName, normalize(null, jsonObject.get("content"), path, ignoredPaths, nsMap).getValue()); } // The conversion to JSON converts <a/> into "" and the semantically equivalent <a></a> into // an empty map, so normalize that here. return new AbstractMap.SimpleEntry<>(elementName, map.isEmpty() ? "" : map); } if (obj instanceof JSONArray) { // Another problem resulting from JSONification: If the array contains elements whose names // are the same before URI expansion, but different after URI expansion, because they use // xmlns attribute that define the namespaces differently, we will screw up. Again, hopefully // that doesn't happen much. The reverse is also true: If the array contains names that are // different before URI expansion, but the same after, we may have a problem, because the // elements will wind up in different JSONArrays as a result of JSONification. We wave our // hands and just assume that the URI expansion of the first element holds for all others. Set<Object> set = new HashSet<>(); String mappedKey = null; for (int i = 0; i < ((JSONArray) obj).length(); ++i) { Map.Entry<String, Object> simpleEntry = normalize(null, ((JSONArray) obj).get(i), path, ignoredPaths, nsMap); if (i == 0) { mappedKey = simpleEntry.getKey(); } set.add(simpleEntry.getValue()); } return new AbstractMap.SimpleEntry<String, Object>(mappedKey, set); } if (obj instanceof Number) { return new AbstractMap.SimpleEntry<String, Object>(null, obj.toString()); } if (obj instanceof Boolean) { return new AbstractMap.SimpleEntry<String, Object>(null, ((Boolean) obj) ? "1" : "0"); } if (obj instanceof String) { // Turn stringified booleans into integers. Both are acceptable as xml boolean values, but // we use "true" and "false" whereas the samples use "1" and "0". if (obj.equals("true")) { return new AbstractMap.SimpleEntry<String, Object>(null, "1"); } if (obj.equals("false")) { return new AbstractMap.SimpleEntry<String, Object>(null, "0"); } String string = obj.toString(); // We use a slightly different datetime format (both legal) than the samples, so normalize // both into Datetime objects. try { return new AbstractMap.SimpleEntry<String, Object>(null, ISODateTimeFormat.dateTime().parseDateTime(string).toDateTime(UTC)); } catch (IllegalArgumentException e) { // It wasn't a DateTime. } try { return new AbstractMap.SimpleEntry<String, Object>(null, ISODateTimeFormat.dateTimeNoMillis().parseDateTime(string).toDateTime(UTC)); } catch (IllegalArgumentException e) { // It wasn't a DateTime. } try { if (!InternetDomainName.isValid(string)) { // It's not a domain name, but it is an InetAddress. Ergo, it's an ip address. return new AbstractMap.SimpleEntry<String, Object>(null, InetAddresses.forString(string)); } } catch (IllegalArgumentException e) { // Not an ip address. } return new AbstractMap.SimpleEntry<String, Object>(null, string); } return new AbstractMap.SimpleEntry<>(null, checkNotNull(obj)); }
From source file:gov.nasa.jpl.mudrod.weblog.pre.CrawlerDetection.java
License:Apache License
private int checkByRate(ESDriver es, String user) { int rate = Integer.parseInt(props.getProperty("sendingrate")); Pattern pattern = Pattern.compile("get (.*?) http/*"); Matcher matcher;/*from ww w . j a va 2s . c om*/ BoolQueryBuilder filterSearch = new BoolQueryBuilder(); filterSearch.must(QueryBuilders.termQuery("IP", user)); AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_minute").field("Time") .dateHistogramInterval(DateHistogramInterval.MINUTE).order(Order.COUNT_DESC); SearchResponse checkRobot = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType) .setQuery(filterSearch).setSize(0).addAggregation(aggregation).execute().actionGet(); Histogram agg = checkRobot.getAggregations().get("by_minute"); List<? extends Histogram.Bucket> botList = agg.getBuckets(); long maxCount = botList.get(0).getDocCount(); if (maxCount >= rate) { return 0; } else { DateTime dt1 = null; int toLast = 0; SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType) .setScroll(new TimeValue(60000)).setQuery(filterSearch).setSize(100).execute().actionGet(); while (true) { for (SearchHit hit : scrollResp.getHits().getHits()) { Map<String, Object> result = hit.getSource(); String logtype = (String) result.get("LogType"); if (logtype.equals("PO.DAAC")) { String request = (String) result.get("Request"); matcher = pattern.matcher(request.trim().toLowerCase()); boolean find = false; while (matcher.find()) { request = matcher.group(1); result.put("RequestUrl", "http://podaac.jpl.nasa.gov" + request); find = true; } if (!find) { result.put("RequestUrl", request); } } else { result.put("RequestUrl", result.get("Request")); } DateTimeFormatter fmt = ISODateTimeFormat.dateTime(); DateTime dt2 = fmt.parseDateTime((String) result.get("Time")); if (dt1 == null) { toLast = 0; } else { toLast = Math.abs(Seconds.secondsBetween(dt1, dt2).getSeconds()); } result.put("ToLast", toLast); IndexRequest ir = new IndexRequest(logIndex, cleanupType).source(result); es.getBulkProcessor().add(ir); dt1 = dt2; } scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(600000)).execute().actionGet(); if (scrollResp.getHits().getHits().length == 0) { break; } } } return 1; }
From source file:gov.nasa.jpl.mudrod.weblog.pre.SessionGenerator.java
License:Apache License
public int genSessionByReferer(ESDriver es, String user, int timeThres) throws ElasticsearchException, IOException { String startTime = null;//from w ww. ja v a 2 s . com int sessionCountIn = 0; BoolQueryBuilder filterSearch = new BoolQueryBuilder(); filterSearch.must(QueryBuilders.termQuery("IP", user)); SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType) .setScroll(new TimeValue(60000)).setQuery(filterSearch).addSort("Time", SortOrder.ASC).setSize(100) .execute().actionGet(); Map<String, Map<String, DateTime>> sessionReqs = new HashMap<>(); String request = ""; String referer = ""; String logType = ""; String id = ""; String ip = user; String indexUrl = "http://podaac.jpl.nasa.gov/"; DateTime time = null; DateTimeFormatter fmt = ISODateTimeFormat.dateTime(); while (scrollResp.getHits().getHits().length != 0) { for (SearchHit hit : scrollResp.getHits().getHits()) { Map<String, Object> result = hit.getSource(); request = (String) result.get("RequestUrl"); referer = (String) result.get("Referer"); logType = (String) result.get("LogType"); time = fmt.parseDateTime((String) result.get("Time")); id = hit.getId(); if ("PO.DAAC".equals(logType)) { if ("-".equals(referer) || referer.equals(indexUrl) || !referer.contains(indexUrl)) { sessionCountIn++; sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>()); sessionReqs.get(ip + "@" + sessionCountIn).put(request, time); update(es, logIndex, this.cleanupType, id, "SessionID", ip + "@" + sessionCountIn); } else { int count = sessionCountIn; int rollbackNum = 0; while (true) { Map<String, DateTime> requests = sessionReqs.get(ip + "@" + count); if (requests == null) { sessionReqs.put(ip + "@" + count, new HashMap<String, DateTime>()); sessionReqs.get(ip + "@" + count).put(request, time); update(es, logIndex, this.cleanupType, id, "SessionID", ip + "@" + count); break; } ArrayList<String> keys = new ArrayList<>(requests.keySet()); boolean bFindRefer = false; for (int i = keys.size() - 1; i >= 0; i--) { rollbackNum++; if (keys.get(i).equalsIgnoreCase(referer)) { bFindRefer = true; // threshold,if time interval > 10* // click num, start a new session if (Math.abs(Seconds.secondsBetween(requests.get(keys.get(i)), time) .getSeconds()) < timeThres * rollbackNum) { sessionReqs.get(ip + "@" + count).put(request, time); update(es, logIndex, this.cleanupType, id, "SessionID", ip + "@" + count); } else { sessionCountIn++; sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>()); sessionReqs.get(ip + "@" + sessionCountIn).put(request, time); update(es, logIndex, this.cleanupType, id, "SessionID", ip + "@" + sessionCountIn); } break; } } if (bFindRefer) { break; } count--; if (count < 0) { sessionCountIn++; sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>()); sessionReqs.get(ip + "@" + sessionCountIn).put(request, time); update(es, props.getProperty(MudrodConstants.ES_INDEX_NAME), this.cleanupType, id, "SessionID", ip + "@" + sessionCountIn); break; } } } } else if ("ftp".equals(logType)) { // may affect computation efficiency Map<String, DateTime> requests = sessionReqs.get(ip + "@" + sessionCountIn); if (requests == null) { sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>()); } else { ArrayList<String> keys = new ArrayList<>(requests.keySet()); int size = keys.size(); if (Math.abs(Seconds.secondsBetween(requests.get(keys.get(size - 1)), time) .getSeconds()) > timeThres) { sessionCountIn += 1; sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>()); } } sessionReqs.get(ip + "@" + sessionCountIn).put(request, time); update(es, logIndex, this.cleanupType, id, "SessionID", ip + "@" + sessionCountIn); } } scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(600000)).execute().actionGet(); } return sessionCountIn; }
From source file:gov.nasa.jpl.mudrod.weblog.pre.SessionGenerator.java
License:Apache License
public void combineShortSessions(ESDriver es, String user, int timeThres) throws ElasticsearchException, IOException { BoolQueryBuilder filterSearch = new BoolQueryBuilder(); filterSearch.must(QueryBuilders.termQuery("IP", user)); String[] indexArr = new String[] { logIndex }; String[] typeArr = new String[] { cleanupType }; int docCount = es.getDocCount(indexArr, typeArr, filterSearch); if (docCount < 3) { deleteInvalid(es, user);// ww w . j a v a 2 s. c o m return; } BoolQueryBuilder filterCheck = new BoolQueryBuilder(); filterCheck.must(QueryBuilders.termQuery("IP", user)).must(QueryBuilders.termQuery("Referer", "-")); SearchResponse checkReferer = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType) .setScroll(new TimeValue(60000)).setQuery(filterCheck).setSize(0).execute().actionGet(); long numInvalid = checkReferer.getHits().getTotalHits(); double invalidRate = numInvalid / docCount; if (invalidRate >= 0.8) { deleteInvalid(es, user); return; } StatsAggregationBuilder statsAgg = AggregationBuilders.stats("Stats").field("Time"); SearchResponse srSession = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType) .setScroll(new TimeValue(60000)).setQuery(filterSearch).addAggregation(AggregationBuilders .terms("Sessions").field("SessionID").size(docCount).subAggregation(statsAgg)) .execute().actionGet(); Terms sessions = srSession.getAggregations().get("Sessions"); List<Session> sessionList = new ArrayList<>(); for (Terms.Bucket session : sessions.getBuckets()) { Stats agg = session.getAggregations().get("Stats"); Session sess = new Session(props, es, agg.getMinAsString(), agg.getMaxAsString(), session.getKey().toString()); sessionList.add(sess); } Collections.sort(sessionList); DateTimeFormatter fmt = ISODateTimeFormat.dateTime(); String last = null; String lastnewID = null; String lastoldID = null; String current = null; for (Session s : sessionList) { current = s.getEndTime(); if (last != null) { if (Seconds.secondsBetween(fmt.parseDateTime(last), fmt.parseDateTime(current)) .getSeconds() < timeThres) { if (lastnewID == null) { s.setNewID(lastoldID); } else { s.setNewID(lastnewID); } QueryBuilder fs = QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("SessionID", s.getID())); SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType) .setScroll(new TimeValue(60000)).setQuery(fs).setSize(100).execute().actionGet(); while (true) { for (SearchHit hit : scrollResp.getHits().getHits()) { if (lastnewID == null) { update(es, logIndex, this.cleanupType, hit.getId(), "SessionID", lastoldID); } else { update(es, logIndex, this.cleanupType, hit.getId(), "SessionID", lastnewID); } } scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(600000)).execute().actionGet(); if (scrollResp.getHits().getHits().length == 0) { break; } } } ; } lastoldID = s.getID(); lastnewID = s.getNewID(); last = current; } }
From source file:gov.nasa.jpl.mudrod.weblog.pre.SessionStatistic.java
License:Apache License
public int processSession(ESDriver es, String sessionId) throws IOException, InterruptedException, ExecutionException { String inputType = cleanupType; String outputType = sessionStats; DateTimeFormatter fmt = ISODateTimeFormat.dateTime(); String min = null;/*from w w w .j av a 2s .co m*/ String max = null; DateTime start = null; DateTime end = null; int duration = 0; float request_rate = 0; int session_count = 0; Pattern pattern = Pattern.compile("get (.*?) http/*"); StatsAggregationBuilder statsAgg = AggregationBuilders.stats("Stats").field("Time"); BoolQueryBuilder filter_search = new BoolQueryBuilder(); filter_search.must(QueryBuilders.termQuery("SessionID", sessionId)); SearchResponse sr = es.getClient().prepareSearch(logIndex).setTypes(inputType).setQuery(filter_search) .addAggregation(statsAgg).execute().actionGet(); Stats agg = sr.getAggregations().get("Stats"); min = agg.getMinAsString(); max = agg.getMaxAsString(); start = fmt.parseDateTime(min); end = fmt.parseDateTime(max); duration = Seconds.secondsBetween(start, end).getSeconds(); int searchDataListRequest_count = 0; int searchDataRequest_count = 0; int searchDataListRequest_byKeywords_count = 0; int ftpRequest_count = 0; int keywords_num = 0; String IP = null; String keywords = ""; String views = ""; String downloads = ""; SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(inputType) .setScroll(new TimeValue(60000)).setQuery(filter_search).setSize(100).execute().actionGet(); while (true) { for (SearchHit hit : scrollResp.getHits().getHits()) { Map<String, Object> result = hit.getSource(); String request = (String) result.get("Request"); String logType = (String) result.get("LogType"); IP = (String) result.get("IP"); Matcher matcher = pattern.matcher(request.trim().toLowerCase()); while (matcher.find()) { request = matcher.group(1); } String datasetlist = "/datasetlist?"; String dataset = "/dataset/"; if (request.contains(datasetlist)) { searchDataListRequest_count++; RequestUrl requestURL = new RequestUrl(); String infoStr = requestURL.getSearchInfo(request) + ","; String info = es.customAnalyzing(props.getProperty("indexName"), infoStr); if (!info.equals(",")) { if (keywords.equals("")) { keywords = keywords + info; } else { String[] items = info.split(","); String[] keywordList = keywords.split(","); for (int m = 0; m < items.length; m++) { if (!Arrays.asList(keywordList).contains(items[m])) { keywords = keywords + items[m] + ","; } } } } } if (request.startsWith(dataset)) { searchDataRequest_count++; if (findDataset(request) != null) { String view = findDataset(request); if ("".equals(views)) { views = view; } else { if (views.contains(view)) { } else { views = views + "," + view; } } } } if ("ftp".equals(logType)) { ftpRequest_count++; String download = ""; String requestLowercase = request.toLowerCase(); if (requestLowercase.endsWith(".jpg") == false && requestLowercase.endsWith(".pdf") == false && requestLowercase.endsWith(".txt") == false && requestLowercase.endsWith(".gif") == false) { download = request; } if ("".equals(downloads)) { downloads = download; } else { if (downloads.contains(download)) { } else { downloads = downloads + "," + download; } } } } scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (scrollResp.getHits().getHits().length == 0) { break; } } if (!keywords.equals("")) { keywords_num = keywords.split(",").length; } if (searchDataListRequest_count != 0 && searchDataListRequest_count <= Integer.parseInt(props.getProperty("searchf")) && searchDataRequest_count != 0 && searchDataRequest_count <= Integer.parseInt(props.getProperty("viewf")) && ftpRequest_count <= Integer.parseInt(props.getProperty("downloadf"))) { String sessionURL = props.getProperty("SessionPort") + props.getProperty("SessionUrl") + "?sessionid=" + sessionId + "&sessionType=" + outputType + "&requestType=" + inputType; session_count = 1; IndexRequest ir = new IndexRequest(logIndex, outputType).source(jsonBuilder().startObject() .field("SessionID", sessionId).field("SessionURL", sessionURL).field("Duration", duration) .field("Number of Keywords", keywords_num).field("Time", min).field("End_time", max) .field("searchDataListRequest_count", searchDataListRequest_count) .field("searchDataListRequest_byKeywords_count", searchDataListRequest_byKeywords_count) .field("searchDataRequest_count", searchDataRequest_count) .field("keywords", es.customAnalyzing(logIndex, keywords)).field("views", views) .field("downloads", downloads).field("request_rate", request_rate).field("Comments", "") .field("Validation", 0).field("Produceby", 0).field("Correlation", 0).field("IP", IP) .endObject()); es.getBulkProcessor().add(ir); } return session_count; }
From source file:gsonjodatime.DateMidnightConverter.java
License:Open Source License
/** * Gson invokes this call-back method during deserialization when it encounters a field of the * specified type. <p>//from w w w.j av a 2 s . co m * * In the implementation of this call-back method, you should consider invoking * {@link com.google.gson.JsonDeserializationContext#deserialize(com.google.gson.JsonElement, java.lang.reflect.Type)} method to create objects * for any non-trivial field of the returned object. However, you should never invoke it on the * the same type passing {@code json} since that will cause an infinite loop (Gson will call your * call-back method again). * @param json The Json data being deserialized * @param typeOfT The type of the Object to deserialize to * @return a deserialized object of the specified type typeOfT which is a subclass of {@code T} * @throws com.google.gson.JsonParseException if json is not in the expected format of {@code typeOfT} */ @Override public DateMidnight deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { // Do not try to deserialize null or empty values if (json.getAsString() == null || json.getAsString().isEmpty()) { return null; } final DateTimeFormatter fmt = ISODateTimeFormat.dateTime(); return new DateMidnight(fmt.parseDateTime(json.getAsString())); }