org.ossmeter.platform.communicationchannel.nntp.local.NNTPDownloader.java Source code

Java tutorial

Introduction

Here is the source code for org.ossmeter.platform.communicationchannel.nntp.local.NNTPDownloader.java

Source

/*******************************************************************************
 * Copyright (c) 2014 OSSMETER Partners.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    Yannis Korkontzelos - Implementation.
 *******************************************************************************/
package org.ossmeter.platform.communicationchannel.nntp.local;

import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.commons.net.nntp.Article;
import org.apache.commons.net.nntp.NNTPClient;
import org.apache.commons.net.nntp.NewsgroupInfo;
import org.ossmeter.platform.communicationchannel.nntp.local.model.ArticleData;
import org.ossmeter.platform.communicationchannel.nntp.local.model.Messages;
import org.ossmeter.platform.communicationchannel.nntp.local.model.NewsgroupData;
import org.ossmeter.repository.model.cc.nntp.NntpNewsGroup;

import com.mongodb.DB;
import com.mongodb.Mongo;

public class NNTPDownloader {

    private final static int RETRIEVAL_STEP = 250;

    public static void main(String[] args) {

        NntpNewsGroup newsgroup = new NntpNewsGroup();
        newsgroup.setName("eclipsePlatform");
        newsgroup.setUrl("news.eclipse.org/eclipse.platform");
        newsgroup.setAuthenticationRequired(true);
        newsgroup.setUsername("exquisitus");
        newsgroup.setPassword("flinder1f7");
        newsgroup.setPort(80);
        newsgroup.setInterval(10000);

        NNTPDownloader downloader = new NNTPDownloader();
        try {
            downloader.downloadMessages(newsgroup);
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    public void downloadMessages(NntpNewsGroup newsgroup) throws Exception {
        final long startTime = System.currentTimeMillis();
        long previousTime = startTime;
        previousTime = printTimeMessage(startTime, previousTime, "Download started");

        NNTPClient nntpClient = NntpUtil.connectToNntpServer(newsgroup);

        NewsgroupInfo newsgroupInfo = NntpUtil.selectNewsgroup(nntpClient, newsgroup);

        int lastArticleChecked = newsgroupInfo.getFirstArticle();
        previousTime = printTimeMessage(startTime, previousTime,
                "First message in newsgroup:\t" + lastArticleChecked);

        int lastArticle = newsgroupInfo.getLastArticle();
        previousTime = printTimeMessage(startTime, previousTime, "Last message in newsgroup:\t" + lastArticle);

        previousTime = printTimeMessage(startTime, previousTime,
                "Articles in newsgroup:\t" + newsgroupInfo.getArticleCount());
        System.err.println();

        Mongo mongo = new Mongo();
        DB db = mongo.getDB(newsgroup.getName() + "LocalStorage");
        Messages dbMessages = new Messages(db);

        NewsgroupData newsgroupData = dbMessages.getNewsgroup().findOneByName(newsgroup.getName());
        if (newsgroupData != null) {
            int newsgroupLastArticleChecked = Integer.parseInt(newsgroupData.getLastArticleChecked());
            if (newsgroupLastArticleChecked > lastArticleChecked) {
                lastArticleChecked = newsgroupLastArticleChecked;
            }
            previousTime = printTimeMessage(startTime, previousTime,
                    "Last article checked set to:\t" + lastArticleChecked);
        } else {
            newsgroupData = new NewsgroupData();
            newsgroupData.setName(newsgroup.getName());
            newsgroupData.setUrl(newsgroup.getUrl());
            newsgroupData.setAuthenticationRequired(newsgroup.getAuthenticationRequired());
            newsgroupData.setUsername(newsgroup.getUsername());
            newsgroupData.setPassword(newsgroup.getPassword());
            newsgroupData.setPort(newsgroup.getPort());
            newsgroupData.setInterval(newsgroup.getInterval());
            newsgroupData.setFirstArticle(lastArticleChecked + "");
            dbMessages.getNewsgroup().add(newsgroupData);
        }

        int retrievalStep = RETRIEVAL_STEP;
        while (lastArticleChecked < lastArticle) {
            if (lastArticleChecked + retrievalStep > lastArticle) {
                retrievalStep = lastArticle - lastArticleChecked;
            }
            Article[] articles = NntpUtil.getArticleInfo(nntpClient, lastArticleChecked + 1,
                    lastArticleChecked + retrievalStep);
            if (articles.length > 0) {
                Article lastArticleRetrieved = articles[articles.length - 1];
                lastArticleChecked = lastArticleRetrieved.getArticleNumber();
                newsgroupData.setLastArticleChecked(lastArticleChecked + "");
            }
            previousTime = printTimeMessage(startTime, previousTime,
                    "downloaded:\t" + articles.length + " nntp articles");
            previousTime = printTimeMessage(startTime, previousTime, "downloading contents\t");

            for (Article article : articles) {
                ArticleData articleData = new ArticleData();
                articleData.setUrl(newsgroup.getUrl());
                articleData.setArticleNumber(article.getArticleNumber());
                articleData.setArticleId(article.getArticleId());
                articleData.setDate(article.getDate());
                articleData.setFrom(article.getFrom());
                articleData.setSubject(article.getSubject());
                for (String referenceId : article.getReferences())
                    articleData.getReferences().add(referenceId);
                articleData.setBody(NntpUtil.getArticleBody(nntpClient, article.getArticleNumber()));
                dbMessages.getArticles().add(articleData);
            }
            dbMessages.sync();
            previousTime = printTimeMessage(startTime, previousTime, "stored:\t" + dbMessages.getArticles().size()
                    + " / " + newsgroupInfo.getArticleCount() + " nntp articles sofar");
            System.err.println();
        }
        nntpClient.disconnect();
        dbMessages.sync();
        previousTime = printTimeMessage(startTime, previousTime, "stored:\t" + dbMessages.getArticles().size()
                + " / " + newsgroupInfo.getArticleCount() + " nntp articles");
    }

    public String getContents(NntpNewsGroup newsgroup, int articleNumber) throws Exception {
        //      System.out.print(" " + articleNumber);
        NNTPClient nntpClient = NntpUtil.connectToNntpServer(newsgroup);
        NntpUtil.selectNewsgroup(nntpClient, newsgroup);
        String contents = NntpUtil.getArticleBody(nntpClient, articleNumber);
        nntpClient.disconnect();
        return contents;
    }

    private long printTimeMessage(long startTime, long previousTime, String message) {
        long currentTime = System.currentTimeMillis();
        System.err
                .println(time(currentTime - previousTime) + "\t" + time(currentTime - startTime) + "\t" + message);
        return currentTime;
    }

    private String time(long timeInMS) {
        return DurationFormatUtils.formatDuration(timeInMS, "HH:mm:ss,SSS");
    }

}