Example usage for org.joda.time.format ISODateTimeFormat dateTime

List of usage examples for org.joda.time.format ISODateTimeFormat dateTime

  1. HOME
  2. Java
  3. org.joda.time
  4. org.joda.time.format.*
  5. ISODateTimeFormat
  6. dateTime

Introduction

In this page you can find the example usage for org.joda.time.format ISODateTimeFormat dateTime.

Prototype

public static DateTimeFormatter dateTime()

Source Link

Document

Returns a formatter that combines a full date and time, separated by a 'T' (yyyy-MM-dd'T'HH:mm:ss.SSSZZ).

Usage

From source file:divconq.interchange.mws.MwsServices.java

License:Open Source License

public FuncResult<XElement> execute(String path, String action, Map<String, String> params) {
    FuncResult<XElement> res = new FuncResult<>();

    try {/*w ww.ja v  a 2 s  .  co m*/
        URI ep = this.endpoint.resolve(path);

        URL url = ep.toURL();

        String urlParameters = "";

        Map<String, String> sorted = new TreeMap<String, String>();
        sorted.put("Action", action);
        sorted.put("AWSAccessKeyId", this.awsKeyId);
        sorted.put("SellerId", this.awsSeller);

        sorted.put("SignatureMethod", "HmacSHA256");
        sorted.put("SignatureVersion", "2");
        sorted.put("Timestamp", ISODateTimeFormat.dateTime().withZoneUTC().print(System.currentTimeMillis()));
        sorted.put("Version", "2013-09-01");

        sorted.putAll(params);

        boolean firstparam = true;

        for (Entry<String, String> pair : sorted.entrySet()) {
            if (firstparam)
                firstparam = false;
            else
                urlParameters += "&";

            urlParameters += Util.urlEncode(pair.getKey(), false) + "="
                    + Util.urlEncode(pair.getValue(), false);
        }

        String algorithm = "HmacSHA256";
        String stringToSign = null;

        StringBuilder data = new StringBuilder();

        data.append("POST\n");

        data.append(ep.getHost().toLowerCase());

        if (!Util.usesStandardPort(ep)) {
            data.append(":");
            data.append(ep.getPort());
        }

        data.append("\n");

        data.append(Util.urlEncode(ep.getPath(), true) + "\n" + urlParameters);

        stringToSign = data.toString();

        String x = Util.sign(stringToSign, this.awsSecretKey, algorithm);

        //System.out.println("Got: " + x);

        urlParameters += "&Signature=" + Util.urlEncode(x, false);

        res.debug("Sending 'POST' request to URL : " + ep);
        HttpsURLConnection con = (HttpsURLConnection) url.openConnection();

        con.setRequestMethod("POST");
        con.setRequestProperty("User-Agent", "DivConq/1.0 (Language=Java/8)");

        // Send post request
        con.setDoOutput(true);
        DataOutputStream wr = new DataOutputStream(con.getOutputStream());
        wr.writeBytes(urlParameters);
        wr.flush();
        wr.close();

        int responseCode = con.getResponseCode();
        res.debug("Response Code : " + responseCode);

        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));

        String inputLine;
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null)
            response.append(inputLine);

        in.close();

        FuncResult<XElement> xres = XmlReader.parse(response.toString(), false);

        res.setResult(xres.getResult());
    } catch (Exception x) {
        res.error("Error loading result" + x);
    }

    return res;
}

From source file:dk.nsi.stamdata.cpr.pvit.proxy.CprAbbsBodyBuilder.java

License:Mozilla Public License

public String createCprAbbsSoapBody(DateTime since) {
    DateTimeFormatter formatter = ISODateTimeFormat.dateTime();
    String sinceAsString = formatter.print(since);

    String sinceTag = String.format("<%s:since>%s</%s:since>", ABBRIVIATION, sinceAsString, ABBRIVIATION);

    return openTag(ABBRIVIATION, NAMESPACE, TAG_NAME, sinceTag);
}

From source file:dk.nsi.stamdata.replication.webservice.RecordXmlGenerator.java

License:Mozilla Public License

public org.w3c.dom.Document generateXml(List<RecordMetadata> records, String register, String datatype,
        DateTime updated) throws TransformerException {
    String stamdataNamespaceUri = STAMDATA_NAMESPACE_URI_PREFIX + register;

    Document document = DocumentHelper.createDocument();
    document.setXMLEncoding("utf-8");

    Element root = document.addElement("atom:feed", ATOM_NAMESPACE_URI);

    addElement(root, ATOM_NAMESPACE_URI, "atom:id",
            String.format("tag:nsi.dk,2011:%s/%s/v1", register, datatype));
    addElement(root, ATOM_NAMESPACE_URI, "atom:updated", AtomDate.toString(updated.toDate()));
    addElement(root, ATOM_NAMESPACE_URI, "atom:title", "Stamdata Registry Feed");
    Element author = addElement(root, ATOM_NAMESPACE_URI, "atom:author", null);
    addElement(author, ATOM_NAMESPACE_URI, "atom:name", "National Sundheds IT");

    for (RecordMetadata metadata : records) {
        Element entry = addElement(root, ATOM_NAMESPACE_URI, "atom:entry", null);
        String atomId = String.format("tag:nsi.dk,2011:%s/%s/v1/%d%07d", register, datatype,
                metadata.getModifiedDate().getMillis(), metadata.getPid());
        addElement(entry, ATOM_NAMESPACE_URI, "atom:id", atomId);
        addElement(entry, ATOM_NAMESPACE_URI, "atom:title", null);

        addElement(entry, ATOM_NAMESPACE_URI, "atom:updated",
                AtomDate.toString(metadata.getModifiedDate().toDate()));

        Element content = addElement(entry, ATOM_NAMESPACE_URI, "atom:content", null);
        content.addAttribute("type", "application/xml");

        Element recordElement = addElement(content, stamdataNamespaceUri, datatype, null);
        for (FieldSpecification fieldSpecification : recordSpecification.getFieldSpecs()) {
            addElement(recordElement, stamdataNamespaceUri, fieldSpecification.name,
                    valueAsString(metadata.getRecord(), fieldSpecification));
        }// w ww .j  av a 2s.  c o m
        DateTimeFormatter formatter = ISODateTimeFormat.dateTime();
        addElement(recordElement, stamdataNamespaceUri, "ValidFrom",
                metadata.getValidFrom().toString(formatter));
        if (metadata.getValidTo() == null) {
            addElement(recordElement, stamdataNamespaceUri, "ValidTo", END_OF_TIME.toString(formatter));
        } else {
            addElement(recordElement, stamdataNamespaceUri, "ValidTo",
                    metadata.getValidTo().toString(formatter));
        }
        addElement(recordElement, stamdataNamespaceUri, "ModifiedDate",
                metadata.getModifiedDate().toString(formatter));
    }

    return convertToW3C(document);
}

From source file:edu.gslis.ts.DumpThriftData.java

License:Apache License

/**
 * @param thriftFile/*  ww w.j a  va2  s .  c o m*/
 */
public void filter(File infile, String parser, GQuery gquery, CollectionStats bgstats) {
    try {
        InputStream in = null;

        if (infile.getName().endsWith(".gz"))
            in = new GZIPInputStream(new FileInputStream(infile));
        else if (infile.getName().endsWith("xz"))
            in = new XZInputStream(new FileInputStream(infile));
        else {
            System.err.println("Regular FileInputStream");
            in = new FileInputStream(infile);
        }

        TTransport inTransport = new TIOStreamTransport(new BufferedInputStream(in));
        TBinaryProtocol inProtocol = new TBinaryProtocol(inTransport);
        inTransport.open();

        try {
            // Run through items in the thrift file
            while (true) {
                final StreamItem item = new StreamItem();
                item.read(inProtocol);
                if (item.body == null || item.body.clean_visible == null) {
                    System.err.println("Body is null.");
                    continue;
                }

                String streamId = "";
                if (item.stream_id != null) {
                    streamId = item.stream_id;
                }

                String dateTime = "";
                long epochTime = 0;
                if (item.stream_time != null && item.stream_time.zulu_timestamp != null) {
                    dateTime = item.stream_time.zulu_timestamp;
                    DateTimeFormatter dtf = ISODateTimeFormat.dateTime();
                    epochTime = dtf.parseMillis(dateTime);
                }

                String source = "";
                if (item.source != null) {
                    source = item.source;
                }

                Map<String, List<Sentence>> parsers = item.body.sentences;
                List<Sentence> sentenceParser = parsers.get(parser);

                QueryDocScorer scorer = new ScorerDirichlet();
                scorer.setCollectionStats(bgstats);
                scorer.setQuery(gquery);

                List<Double> sentenceScores = new ArrayList<Double>();
                List<String> sentences = new ArrayList<String>();
                String sentencesText = "";
                if (sentenceParser != null && sentenceParser.size() > 0) {

                    for (Sentence s : sentenceParser) {
                        try {
                            List<Token> tokens = s.tokens;
                            String sentence = "";
                            for (Token token : tokens) {
                                String tok = token.token;
                                sentence += tok + " ";
                            }
                            FeatureVector sentenceVector = new FeatureVector(sentence, null);
                            SearchHit sentenceHit = new SearchHit();
                            sentenceHit.setFeatureVector(sentenceVector);
                            sentenceHit.setLength(sentenceVector.getLength());
                            double score = scorer.score(sentenceHit);

                            sentenceScores.add(score);
                            sentences.add(sentence);

                            sentencesText += sentence + "\n";
                        } catch (Exception e) {
                            System.err
                                    .println("Issue with sentence " + sentences.size() + " in doc " + streamId);
                            System.err.println("File: " + infile.getAbsolutePath());
                        }
                    }
                    SearchHit docHit = new SearchHit();
                    docHit.setFeatureVector(new FeatureVector(sentencesText, null));
                    double docscore = scorer.score(docHit);
                    for (int i = 0; i < sentenceScores.size(); i++) {
                        System.out.println(infile.getAbsolutePath() + "\t" + source + "\t" + epochTime + "\t"
                                + streamId + "\t" + docscore + "\t" + i + "\t" + sentenceScores.get(i) + "\t"
                                + sentences.get(i));
                    }
                } else if (sentenceParser == null) {
                    System.err.println("Sentence parser null");
                } else if (sentenceParser.size() == 0) {
                    System.err.println("Sentence length 0");
                } else {
                    System.err.println("Other sentence error.");
                }

            }
        } catch (TTransportException te) {
            if (te.getType() == TTransportException.END_OF_FILE) {
            } else {
                throw te;
            }
        }

        inTransport.close();

    } catch (Exception e) {
        System.err.println("Error processing " + infile.getAbsolutePath() + " " + infile.getName());
        e.printStackTrace();
    }
}

From source file:edu.gslis.ts.ThriftToTREC.java

License:Apache License

/**
 * @param thriftFile//from   www. ja  v  a 2s  . c om
 */
public Map<String, String> filter(File infile, File outfile, String parser) {
    Map<String, String> results = new TreeMap<String, String>();
    try {
        InputStream in = null;

        if (infile.getName().endsWith(".gz"))
            in = new GZIPInputStream(new FileInputStream(infile));
        else if (infile.getName().endsWith("xz"))
            in = new XZInputStream(new FileInputStream(infile));
        else
            in = new FileInputStream(infile);

        TTransport inTransport = new TIOStreamTransport(new BufferedInputStream(in));
        TBinaryProtocol inProtocol = new TBinaryProtocol(inTransport);
        inTransport.open();

        OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(outfile, false), "UTF-8");
        try {
            Charset charset = Charset.forName("UTF-8");
            CharsetDecoder decoder = charset.newDecoder();

            // Run through items in the thrift file
            while (true) {
                final StreamItem item = new StreamItem();
                item.read(inProtocol);
                if (item.body == null || item.body.clean_visible == null) {
                    continue;
                }

                String streamId = "";
                if (item.stream_id != null) {
                    streamId = item.stream_id;
                }

                String dateTime = "";
                long epochTime = 0;
                if (item.stream_time != null && item.stream_time.zulu_timestamp != null) {
                    dateTime = item.stream_time.zulu_timestamp;
                    DateTimeFormatter dtf = ISODateTimeFormat.dateTime();
                    epochTime = dtf.parseMillis(dateTime);
                }

                String source = "";
                if (item.source != null) {
                    source = item.source;
                }

                String url = "";
                if (item.abs_url != null) {
                    url = decoder.decode(item.abs_url).toString();
                }

                Map<String, List<Sentence>> parsers = item.body.sentences;
                List<Sentence> sentenceParser = parsers.get(parser);

                String sentencesText = "";
                int sentenceNum = 0;
                if (sentenceParser != null && sentenceParser.size() > 0) {

                    for (Sentence s : sentenceParser) {
                        List<Token> tokens = s.tokens;
                        String sentence = "";
                        for (Token token : tokens) {
                            String tok = token.token;
                            sentence += tok + " ";
                        }
                        sentencesText += sentenceNum + " " + sentence + "\n";
                        sentenceNum++;
                    }
                }

                try {
                    String hourDayDir = outfile.getName().replace(".txt", "");
                    out.write("<DOC>\n");
                    out.write("<DOCNO>" + streamId + "</DOCNO>\n");
                    out.write("<SOURCE>" + source + "</SOURCE>\n");
                    out.write("<URL>" + url + "</URL>\n");
                    out.write("<DATETIME>" + dateTime + "</DATETIME>\n");
                    out.write("<HOURDAYDIR>" + hourDayDir + "</HOURDAYDIR>\n");
                    out.write("<EPOCH>" + epochTime + "</EPOCH>\n");
                    out.write("<TEXT>\n" + sentencesText + "\n</TEXT>\n");
                    out.write("</DOC>\n");
                } catch (Exception e) {
                    System.out.println("Error processing " + infile.getAbsolutePath() + " " + item.stream_id);
                    e.printStackTrace();
                }

            }
        } catch (TTransportException te) {
            if (te.getType() == TTransportException.END_OF_FILE) {
            } else {
                throw te;
            }
        }
        inTransport.close();
        out.close();

    } catch (Exception e) {
        System.out.println("Error processing " + infile.getName());
        e.printStackTrace();
    }
    return results;
}

From source file:edu.iastate.music.marching.attendance.util.DateTimeConverter.java

License:Open Source License

/**
 * Gson invokes this call-back method during deserialization when it encounters a field of the
 * specified type. <p>//from  w  ww.  j  a v  a  2  s  . c  o  m
 *
 * In the implementation of this call-back method, you should consider invoking
 * {@link JsonDeserializationContext#deserialize(JsonElement, Type)} method to create objects
 * for any non-trivial field of the returned object. However, you should never invoke it on the
 * the same type passing {@code json} since that will cause an infinite loop (Gson will call your
 * call-back method again).
 *
 * @param json The Json data being deserialized
 * @param typeOfT The type of the Object to deserialize to
 * @return a deserialized object of the specified type typeOfT which is a subclass of {@code T}
 * @throws JsonParseException if json is not in the expected format of {@code typeOfT}
 */
@Override
public DateTime deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
        throws JsonParseException {
    final DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
    return fmt.parseDateTime(json.getAsString()).toDateTime(DateTimeZone.UTC);
}

From source file:edu.unc.lib.dl.services.OperationsMessageSender.java

License:Apache License

/**
 * @param msg//ww w. j ava 2  s  .co m
 * @param userid
 * @param pid
 * @return
 */
private Element createAtomEntry(Document msg, String userid, PID contextpid, String operation,
        String messageId) {
    Element entry = new Element("entry", ATOM_NS);
    msg.addContent(entry);
    entry.addContent(new Element("id", ATOM_NS).setText(messageId));
    DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
    String timestamp = fmt.print(DateTimeUtils.currentTimeMillis());
    entry.addContent(new Element("updated", ATOM_NS).setText(timestamp));
    entry.addContent(new Element("author", ATOM_NS).addContent(new Element("name", ATOM_NS).setText(userid))
            .addContent(new Element("uri", ATOM_NS).setText(CDR_MESSAGE_AUTHOR_URI)));
    entry.addContent(new Element("title", ATOM_NS).setText(operation).setAttribute("type", "text"));
    entry.addContent(new Element("summary", ATOM_NS).setText(contextpid.getPid()).setAttribute("type", "text"));
    Element content = new Element("content", ATOM_NS).setAttribute("type", "text/xml");
    entry.addContent(content);
    return content;
}

From source file:esiptestbed.mudrod.weblog.pre.CrawlerDetection.java

License:Apache License

/**
 * Check crawler by request sending rate, which is read from configruation file
 * @throws InterruptedException InterruptedException
 * @throws IOException IOException/*from   w  w w  .  ja va 2  s.co m*/
 */
public void checkByRate() throws InterruptedException, IOException {
    es.createBulkProcessor();

    int rate = Integer.parseInt(props.getProperty("sendingrate"));
    SearchResponse sr = es.getClient().prepareSearch(props.getProperty(MudrodConstants.ES_INDEX_NAME))
            .setTypes(httpType).setQuery(QueryBuilders.matchAllQuery()).setSize(0)
            .addAggregation(AggregationBuilders.terms("Users").field("IP").size(0)).execute().actionGet();
    Terms users = sr.getAggregations().get("Users");

    int userCount = 0;

    Pattern pattern = Pattern.compile("get (.*?) http/*");
    Matcher matcher;
    for (Terms.Bucket entry : users.getBuckets()) {
        QueryBuilder filterSearch = QueryBuilders.boolQuery()
                .must(QueryBuilders.termQuery("IP", entry.getKey()));
        QueryBuilder querySearch = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filterSearch);

        AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_minute").field("Time")
                .interval(DateHistogramInterval.MINUTE).order(Order.COUNT_DESC);
        SearchResponse checkRobot = es.getClient().prepareSearch(props.getProperty("indexName"))
                .setTypes(httpType, ftpType).setQuery(querySearch).setSize(0).addAggregation(aggregation)
                .execute().actionGet();

        Histogram agg = checkRobot.getAggregations().get("by_minute");

        List<? extends Histogram.Bucket> botList = agg.getBuckets();
        long maxCount = botList.get(0).getDocCount();
        if (maxCount >= rate) {
        } else {
            userCount++;
            DateTime dt1 = null;
            int toLast = 0;
            SearchResponse scrollResp = es.getClient().prepareSearch(props.getProperty("indexName"))
                    .setTypes(httpType, ftpType).setScroll(new TimeValue(60000)).setQuery(querySearch)
                    .setSize(100).execute().actionGet();
            while (true) {
                for (SearchHit hit : scrollResp.getHits().getHits()) {
                    Map<String, Object> result = hit.getSource();
                    String logtype = (String) result.get("LogType");
                    if (logtype.equals("PO.DAAC")) {
                        String request = (String) result.get("Request");
                        matcher = pattern.matcher(request.trim().toLowerCase());
                        boolean find = false;
                        while (matcher.find()) {
                            request = matcher.group(1);
                            result.put("RequestUrl", "http://podaac.jpl.nasa.gov" + request);
                            find = true;
                        }
                        if (!find) {
                            result.put("RequestUrl", request);
                        }
                    } else {
                        result.put("RequestUrl", (String) result.get("Request"));
                    }

                    DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
                    DateTime dt2 = fmt.parseDateTime((String) result.get("Time"));

                    if (dt1 == null) {
                        toLast = 0;
                    } else {
                        toLast = Math.abs(Seconds.secondsBetween(dt1, dt2).getSeconds());
                    }
                    result.put("ToLast", toLast);
                    IndexRequest ir = new IndexRequest(props.getProperty("indexName"), cleanupType)
                            .source(result);

                    es.getBulkProcessor().add(ir);
                    dt1 = dt2;
                }

                scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId())
                        .setScroll(new TimeValue(600000)).execute().actionGet();
                if (scrollResp.getHits().getHits().length == 0) {
                    break;
                }
            }

        }
    }
    es.destroyBulkProcessor();
    LOG.info("User count: {}", Integer.toString(userCount));
}

From source file:esiptestbed.mudrod.weblog.pre.SessionGenerator.java

License:Apache License

/**
 * Method to generate session by time threshold and referrer
 * @param timeThres value of time threshold (s)
 * @throws ElasticsearchException ElasticsearchException
 * @throws IOException IOException//from  w w  w. ja v  a2s  . co m
 */
public void genSessionByReferer(int timeThres) throws ElasticsearchException, IOException {
    SearchResponse sr = es.getClient().prepareSearch(props.getProperty("indexName")).setTypes(this.cleanupType)
            .setQuery(QueryBuilders.matchAllQuery()).setSize(0)
            .addAggregation(AggregationBuilders.terms("Users").field("IP").size(0)).execute().actionGet();
    Terms users = sr.getAggregations().get("Users");

    int sessionCount = 0;
    for (Terms.Bucket entry : users.getBuckets()) {

        String startTime = null;
        int sessionCountIn = 0;

        QueryBuilder filterSearch = QueryBuilders.boolQuery()
                .must(QueryBuilders.termQuery("IP", entry.getKey()));
        QueryBuilder querySearch = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filterSearch);

        SearchResponse scrollResp = es.getClient().prepareSearch(props.getProperty("indexName"))
                .setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(querySearch)
                .addSort("Time", SortOrder.ASC).setSize(100).execute().actionGet();

        Map<String, Map<String, DateTime>> sessionReqs = new HashMap<>();
        String request = "";
        String referer = "";
        String logType = "";
        String id = "";
        String ip = entry.getKey().toString();
        String indexUrl = "http://podaac.jpl.nasa.gov/";
        DateTime time = null;
        DateTimeFormatter fmt = ISODateTimeFormat.dateTime();

        while (scrollResp.getHits().getHits().length != 0) {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                Map<String, Object> result = hit.getSource();
                request = (String) result.get("RequestUrl");
                referer = (String) result.get("Referer");
                logType = (String) result.get("LogType");
                time = fmt.parseDateTime((String) result.get("Time"));
                id = hit.getId();

                if (logType.equals("PO.DAAC")) {
                    if (referer.equals("-") || referer.equals(indexUrl) || !referer.contains(indexUrl)) {
                        sessionCount++;
                        sessionCountIn++;
                        sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>());
                        sessionReqs.get(ip + "@" + sessionCountIn).put(request, time);

                        update(props.getProperty("indexName"), this.cleanupType, id, "SessionID",
                                ip + "@" + sessionCountIn);

                    } else {
                        int count = sessionCountIn;
                        int rollbackNum = 0;
                        while (true) {
                            Map<String, DateTime> requests = sessionReqs.get(ip + "@" + count);
                            if (requests == null) {
                                sessionReqs.put(ip + "@" + count, new HashMap<String, DateTime>());
                                sessionReqs.get(ip + "@" + count).put(request, time);
                                update(props.getProperty("indexName"), this.cleanupType, id, "SessionID",
                                        ip + "@" + count);

                                break;
                            }
                            ArrayList<String> keys = new ArrayList<>(requests.keySet());
                            boolean bFindRefer = false;

                            for (int i = keys.size() - 1; i >= 0; i--) {
                                rollbackNum++;
                                if (keys.get(i).equalsIgnoreCase(referer)) {
                                    bFindRefer = true;
                                    // threshold,if time interval > 10*
                                    // click num, start a new session
                                    if (Math.abs(Seconds.secondsBetween(requests.get(keys.get(i)), time)
                                            .getSeconds()) < timeThres * rollbackNum) {
                                        sessionReqs.get(ip + "@" + count).put(request, time);
                                        update(props.getProperty("indexName"), this.cleanupType, id,
                                                "SessionID", ip + "@" + count);
                                    } else {
                                        sessionCount++;
                                        sessionCountIn++;
                                        sessionReqs.put(ip + "@" + sessionCountIn,
                                                new HashMap<String, DateTime>());
                                        sessionReqs.get(ip + "@" + sessionCountIn).put(request, time);
                                        update(props.getProperty("indexName"), this.cleanupType, id,
                                                "SessionID", ip + "@" + sessionCountIn);
                                    }

                                    break;
                                }
                            }

                            if (bFindRefer) {
                                break;
                            }

                            count--;
                            if (count < 0) {

                                sessionCount++;
                                sessionCountIn++;

                                sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>());
                                sessionReqs.get(ip + "@" + sessionCountIn).put(request, time);
                                update(props.getProperty(MudrodConstants.ES_INDEX_NAME), this.cleanupType, id,
                                        "SessionID", ip + "@" + sessionCountIn);

                                break;
                            }
                        }
                    }
                } else if ("ftp".equals(logType)) {

                    // may affect computation efficiency
                    Map<String, DateTime> requests = sessionReqs.get(ip + "@" + sessionCountIn);
                    if (requests == null) {
                        sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>());
                    } else {
                        ArrayList<String> keys = new ArrayList<>(requests.keySet());
                        int size = keys.size();
                        if (Math.abs(Seconds.secondsBetween(requests.get(keys.get(size - 1)), time)
                                .getSeconds()) > timeThres) {
                            sessionCount += 1;
                            sessionCountIn += 1;
                            sessionReqs.put(ip + "@" + sessionCountIn, new HashMap<String, DateTime>());
                        }
                    }
                    sessionReqs.get(ip + "@" + sessionCountIn).put(request, time);
                    update(props.getProperty("indexName"), this.cleanupType, id, "SessionID",
                            ip + "@" + sessionCountIn);
                }
            }

            scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId())
                    .setScroll(new TimeValue(600000)).execute().actionGet();
        }
    }
}

From source file:esiptestbed.mudrod.weblog.pre.SessionGenerator.java

License:Apache License

public void combineShortSessions(int Timethres) throws ElasticsearchException, IOException {
    SearchResponse sr = es.getClient().prepareSearch(props.getProperty("indexName")).setTypes(this.cleanupType)
            .setQuery(QueryBuilders.matchAllQuery())
            .addAggregation(AggregationBuilders.terms("Users").field("IP").size(0)).execute().actionGet();
    Terms users = sr.getAggregations().get("Users");

    for (Terms.Bucket entry : users.getBuckets()) {
        QueryBuilder filterAll = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("IP", entry.getKey()));
        QueryBuilder queryAll = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filterAll);
        SearchResponse checkAll = es.getClient().prepareSearch(props.getProperty("indexName"))
                .setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(queryAll).setSize(0)
                .execute().actionGet();//  ww  w  .j ava2 s  .  c o m

        long all = checkAll.getHits().getTotalHits();

        QueryBuilder filterCheck = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("IP", entry.getKey()))
                .must(QueryBuilders.termQuery("Referer", "-"));
        QueryBuilder queryCheck = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filterCheck);
        SearchResponse checkReferer = es.getClient().prepareSearch(props.getProperty("indexName"))
                .setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(queryCheck).setSize(0)
                .execute().actionGet();

        long numInvalid = checkReferer.getHits().getTotalHits();

        double invalidRate = (float) (numInvalid / all);

        if (invalidRate >= 0.8 || all < 3) {
            deleteInvalid(entry.getKey().toString());
            continue;
        }

        QueryBuilder filterSearch = QueryBuilders.boolQuery()
                .must(QueryBuilders.termQuery("IP", entry.getKey()));
        QueryBuilder querySearch = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filterSearch);

        MetricsAggregationBuilder statsAgg = AggregationBuilders.stats("Stats").field("Time");
        SearchResponse sr_session = es
                .getClient().prepareSearch(props.getProperty("indexName")).setTypes(this.cleanupType)
                .setScroll(new TimeValue(60000)).setQuery(querySearch).addAggregation(AggregationBuilders
                        .terms("Sessions").field("SessionID").size(0).subAggregation(statsAgg))
                .execute().actionGet();

        Terms sessions = sr_session.getAggregations().get("Sessions");

        List<Session> sessionList = new ArrayList<>();
        for (Terms.Bucket session : sessions.getBuckets()) {
            Stats agg = session.getAggregations().get("Stats");
            Session sess = new Session(props, es, agg.getMinAsString(), agg.getMaxAsString(),
                    session.getKey().toString());
            sessionList.add(sess);
        }

        Collections.sort(sessionList);
        DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
        String last = null;
        String lastnewID = null;
        String lastoldID = null;
        String current = null;
        for (Session s : sessionList) {
            current = s.getEndTime();
            if (last != null) {
                if (Seconds.secondsBetween(fmt.parseDateTime(last), fmt.parseDateTime(current))
                        .getSeconds() < Timethres) {
                    if (lastnewID == null) {
                        s.setNewID(lastoldID);
                    } else {
                        s.setNewID(lastnewID);
                    }

                    QueryBuilder fs = QueryBuilders.boolQuery()
                            .must(QueryBuilders.termQuery("SessionID", s.getID()));
                    QueryBuilder qs = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), fs);
                    SearchResponse scrollResp = es.getClient().prepareSearch(props.getProperty("indexName"))
                            .setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(qs)
                            .setSize(100).execute().actionGet();
                    while (true) {
                        for (SearchHit hit : scrollResp.getHits().getHits()) {
                            if (lastnewID == null) {
                                update(props.getProperty("indexName"), this.cleanupType, hit.getId(),
                                        "SessionID", lastoldID);
                            } else {
                                update(props.getProperty("indexName"), this.cleanupType, hit.getId(),
                                        "SessionID", lastnewID);
                            }
                        }

                        scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId())
                                .setScroll(new TimeValue(600000)).execute().actionGet();
                        if (scrollResp.getHits().getHits().length == 0) {
                            break;
                        }
                    }
                }
                ;
            }
            lastoldID = s.getID();
            lastnewID = s.getNewID();
            last = current;
        }
    }
}