List of usage examples for org.apache.commons.net.nntp Article getSubject
public String getSubject()
From source file:examples.nntp.ExtendedNNTPOps.java
private void demo(String host, String user, String password) { try {//from w w w.j a va2s. c o m client.connect(host); // AUTHINFO USER/AUTHINFO PASS if (user != null && password != null) { boolean success = client.authenticate(user, password); if (success) { System.out.println("Authentication succeeded"); } else { System.out.println("Authentication failed, error =" + client.getReplyString()); } } // XOVER NewsgroupInfo testGroup = new NewsgroupInfo(); client.selectNewsgroup("alt.test", testGroup); long lowArticleNumber = testGroup.getFirstArticleLong(); long highArticleNumber = lowArticleNumber + 100; Iterable<Article> articles = client.iterateArticleInfo(lowArticleNumber, highArticleNumber); for (Article article : articles) { if (article.isDummy()) { // Subject will contain raw response System.out.println("Could not parse: " + article.getSubject()); } else { System.out.println(article.getSubject()); } } // LIST ACTIVE NewsgroupInfo[] fanGroups = client.listNewsgroups("alt.fan.*"); for (NewsgroupInfo fanGroup : fanGroups) { System.out.println(fanGroup.getNewsgroup()); } } catch (IOException e) { e.printStackTrace(); } }
From source file:net.longfalcon.newsj.Blacklist.java
public boolean isBlackListed(Article article, Group group) { boolean isBlackListed = false; String[] fields = new String[4]; fields[0] = ""; fields[Defaults.BLACKLIST_FIELD_SUBJECT] = article.getSubject(); fields[Defaults.BLACKLIST_FIELD_FROM] = article.getFrom(); fields[Defaults.BLACKLIST_FIELD_MESSAGEID] = article.getArticleId(); List<BinaryBlacklistEntry> blacklistEntries = getCachedBlacklist(); for (BinaryBlacklistEntry blacklistEntry : blacklistEntries) { Pattern groupPattern = Pattern.compile("^" + blacklistEntry.getGroupName() + "$", Pattern.CASE_INSENSITIVE); Matcher groupNameMatcher = groupPattern.matcher(group.getName()); if (groupNameMatcher.matches()) { Pattern blacklistPattern = Pattern.compile(blacklistEntry.getRegex(), Pattern.CASE_INSENSITIVE); String matcherText = fields[blacklistEntry.getMsgCol()]; Matcher blacklistMatcher = blacklistPattern.matcher(matcherText); if (blacklistEntry.getOpType() == 1) { // remove what matches if (blacklistMatcher.find()) { isBlackListed = true; }/*from w ww. jav a2 s . c o m*/ } else if (blacklistEntry.getOpType() == 2) { // remove all that doesnt match if (!blacklistMatcher.find()) { isBlackListed = true; } } } } return isBlackListed; }
From source file:com.almarsoft.GroundhogReader.lib.ServerManager.java
private long insertArticleInDB(Article articleInfo, long articleNumber, String decodedfrom, String charset, SQLiteDatabase catchedDB) throws UsenetReaderException { // Get the reference list as a string instead of as an array of strings // for insertion into the DB String[] references = articleInfo.getReferences(); StringBuilder references_buff = new StringBuilder(); references_buff.append(""); int referencesLen = references.length; for (int i = 0; i < referencesLen; i++) { if (i == (referencesLen - 1)) { references_buff.append(references[i]); } else {//from ww w . j ava2 s . c o m references_buff.append(references[i]); references_buff.append(" "); } } String finalRefs = references_buff.toString(); //String finalSubject = MessageTextProcessor.decodeHeaderInArticleInfo(articleInfo.getSubject(), charset); String subject = articleInfo.getSubject(); subject = subject.replaceAll(" +", " "); // Now insert the Article into the DB return DBUtils.insertArticleToGroupID(mGroupID, articleInfo, finalRefs, decodedfrom, subject, mContext, catchedDB); }
From source file:org.apache.common.net.examples.nntp.ExtendedNNTPOps.java
private void demo(String host, String user, String password) { try {// www. j av a2 s .co m client.connect(host); // AUTHINFO USER/AUTHINFO PASS if (user != null && password != null) { boolean success = client.authenticate(user, password); if (success) { System.out.println("Authentication succeeded"); } else { System.out.println("Authentication failed, error =" + client.getReplyString()); } } // XOVER NewsgroupInfo testGroup = new NewsgroupInfo(); client.selectNewsgroup("alt.test", testGroup); long lowArticleNumber = testGroup.getFirstArticleLong(); long highArticleNumber = lowArticleNumber + 100; Iterable<Article> articles = client.iterateArticleInfo(lowArticleNumber, highArticleNumber); for (Article article : articles) { if (article.isDummy()) { // Subject will contain raw response System.out.println("Could not parse: " + article.getSubject()); } else { System.out.println(article.getSubject()); } } // LIST ACTIVE NewsgroupInfo[] fanGroups = client.listNewsgroups("alt.fan.*"); for (int i = 0; i < fanGroups.length; ++i) { System.out.println(fanGroups[i].getNewsgroup()); } } catch (IOException e) { e.printStackTrace(); } }
From source file:org.ossmeter.platform.communicationchannel.nntp.local.NNTPDownloader.java
public void downloadMessages(NntpNewsGroup newsgroup) throws Exception { final long startTime = System.currentTimeMillis(); long previousTime = startTime; previousTime = printTimeMessage(startTime, previousTime, "Download started"); NNTPClient nntpClient = NntpUtil.connectToNntpServer(newsgroup); NewsgroupInfo newsgroupInfo = NntpUtil.selectNewsgroup(nntpClient, newsgroup); int lastArticleChecked = newsgroupInfo.getFirstArticle(); previousTime = printTimeMessage(startTime, previousTime, "First message in newsgroup:\t" + lastArticleChecked); int lastArticle = newsgroupInfo.getLastArticle(); previousTime = printTimeMessage(startTime, previousTime, "Last message in newsgroup:\t" + lastArticle); previousTime = printTimeMessage(startTime, previousTime, "Articles in newsgroup:\t" + newsgroupInfo.getArticleCount()); System.err.println();// w ww .ja v a2 s . c om Mongo mongo = new Mongo(); DB db = mongo.getDB(newsgroup.getName() + "LocalStorage"); Messages dbMessages = new Messages(db); NewsgroupData newsgroupData = dbMessages.getNewsgroup().findOneByName(newsgroup.getName()); if (newsgroupData != null) { int newsgroupLastArticleChecked = Integer.parseInt(newsgroupData.getLastArticleChecked()); if (newsgroupLastArticleChecked > lastArticleChecked) { lastArticleChecked = newsgroupLastArticleChecked; } previousTime = printTimeMessage(startTime, previousTime, "Last article checked set to:\t" + lastArticleChecked); } else { newsgroupData = new NewsgroupData(); newsgroupData.setName(newsgroup.getName()); newsgroupData.setUrl(newsgroup.getUrl()); newsgroupData.setAuthenticationRequired(newsgroup.getAuthenticationRequired()); newsgroupData.setUsername(newsgroup.getUsername()); newsgroupData.setPassword(newsgroup.getPassword()); newsgroupData.setPort(newsgroup.getPort()); newsgroupData.setInterval(newsgroup.getInterval()); newsgroupData.setFirstArticle(lastArticleChecked + ""); dbMessages.getNewsgroup().add(newsgroupData); } int retrievalStep = RETRIEVAL_STEP; while (lastArticleChecked < lastArticle) { if (lastArticleChecked + retrievalStep > lastArticle) { retrievalStep = lastArticle - lastArticleChecked; } Article[] articles = NntpUtil.getArticleInfo(nntpClient, lastArticleChecked + 1, lastArticleChecked + retrievalStep); if (articles.length > 0) { Article lastArticleRetrieved = articles[articles.length - 1]; lastArticleChecked = lastArticleRetrieved.getArticleNumber(); newsgroupData.setLastArticleChecked(lastArticleChecked + ""); } previousTime = printTimeMessage(startTime, previousTime, "downloaded:\t" + articles.length + " nntp articles"); previousTime = printTimeMessage(startTime, previousTime, "downloading contents\t"); for (Article article : articles) { ArticleData articleData = new ArticleData(); articleData.setUrl(newsgroup.getUrl()); articleData.setArticleNumber(article.getArticleNumber()); articleData.setArticleId(article.getArticleId()); articleData.setDate(article.getDate()); articleData.setFrom(article.getFrom()); articleData.setSubject(article.getSubject()); for (String referenceId : article.getReferences()) articleData.getReferences().add(referenceId); articleData.setBody(NntpUtil.getArticleBody(nntpClient, article.getArticleNumber())); dbMessages.getArticles().add(articleData); } dbMessages.sync(); previousTime = printTimeMessage(startTime, previousTime, "stored:\t" + dbMessages.getArticles().size() + " / " + newsgroupInfo.getArticleCount() + " nntp articles sofar"); System.err.println(); } nntpClient.disconnect(); dbMessages.sync(); previousTime = printTimeMessage(startTime, previousTime, "stored:\t" + dbMessages.getArticles().size() + " / " + newsgroupInfo.getArticleCount() + " nntp articles"); }
From source file:org.ossmeter.platform.communicationchannel.nntp.NntpManager.java
@Override public CommunicationChannelDelta getDelta(DB db, NntpNewsGroup newsgroup, Date date) throws Exception { NNTPClient nntpClient = NntpUtil.connectToNntpServer(newsgroup); NewsgroupInfo newsgroupInfo = NntpUtil.selectNewsgroup(nntpClient, newsgroup); int lastArticle = newsgroupInfo.getLastArticle(); // The following statement is not really needed, but I added it to speed up running, // in the date is far latter than the first day of the newsgroup. // if (Integer.parseInt(newsgroup.getLastArticleChecked())<134500) // newsgroup.setLastArticleChecked("134500"); //137500"); String lac = newsgroup.getLastArticleChecked(); if (lac == null || lac.equals("") || lac.equals("null")) lac = "-1"; int lastArticleChecked = Integer.parseInt(lac); if (lastArticleChecked < 0) lastArticleChecked = newsgroupInfo.getFirstArticle(); // FIXME: certain eclipse newsgroups return 0 for both FirstArticle and LastArticle which causes exceptions if (lastArticleChecked == 0) return null; CommunicationChannelDelta delta = new CommunicationChannelDelta(); delta.setNewsgroup(newsgroup);//w w w .java2 s .c o m int retrievalStep = RETRIEVAL_STEP; Boolean dayCompleted = false; while (!dayCompleted) { if (lastArticleChecked + retrievalStep > lastArticle) { retrievalStep = lastArticle - lastArticleChecked; dayCompleted = true; } Article[] articles; Date articleDate = date; // The following loop discards messages for days earlier than the required one. do { articles = NntpUtil.getArticleInfo(nntpClient, lastArticleChecked + 1, lastArticleChecked + retrievalStep); if (articles.length > 0) { Article lastArticleRetrieved = articles[articles.length - 1]; java.util.Date javaArticleDate = NntpUtil.parseDate(lastArticleRetrieved.getDate()); articleDate = new Date(javaArticleDate); if (date.compareTo(articleDate) > 0) lastArticleChecked = lastArticleRetrieved.getArticleNumber(); } } while (date.compareTo(articleDate) > 0); for (Article article : articles) { java.util.Date javaArticleDate = NntpUtil.parseDate(article.getDate()); if (javaArticleDate != null) { articleDate = new Date(javaArticleDate); if (date.compareTo(articleDate) < 0) { dayCompleted = true; // System.out.println("dayCompleted"); } else if (date.compareTo(articleDate) == 0) { CommunicationChannelArticle communicationChannelArticle = new CommunicationChannelArticle(); communicationChannelArticle.setArticleId(article.getArticleId()); communicationChannelArticle.setArticleNumber(article.getArticleNumber()); communicationChannelArticle.setDate(javaArticleDate); // I haven't seen any messageThreadIds on NNTP servers, yet. // communicationChannelArticle.setMessageThreadId(article.messageThreadId()); NntpNewsGroup newNewsgroup = new NntpNewsGroup(); newNewsgroup.setUrl(newsgroup.getUrl()); newNewsgroup.setAuthenticationRequired(newsgroup.getAuthenticationRequired()); newNewsgroup.setUsername(newsgroup.getUsername()); newNewsgroup.setPassword(newsgroup.getPassword()); newNewsgroup.setNewsGroupName(newsgroup.getNewsGroupName()); newNewsgroup.setPort(newsgroup.getPort()); newNewsgroup.setInterval(newsgroup.getInterval()); communicationChannelArticle.setNewsgroup(newNewsgroup); communicationChannelArticle.setReferences(article.getReferences()); communicationChannelArticle.setSubject(article.getSubject()); communicationChannelArticle.setUser(article.getFrom()); communicationChannelArticle .setText(getContents(db, newNewsgroup, communicationChannelArticle)); delta.getArticles().add(communicationChannelArticle); lastArticleChecked = article.getArticleNumber(); // System.out.println("dayNOTCompleted"); } else { //TODO: In this case, there are unprocessed articles whose date is earlier than the date requested. // This means that the deltas of those article dates are incomplete, // i.e. the deltas did not contain all articles of those dates. } } else { // If an article has no correct date, then ignore it System.err.println("\t\tUnparsable article date: " + article.getDate()); } } } nntpClient.disconnect(); newsgroup.setLastArticleChecked(lastArticleChecked + ""); System.out.println( "delta (" + date.toString() + ") contains:\t" + delta.getArticles().size() + " nntp articles"); return delta; }