Java tutorial
/** * License Agreement for OpenSearchServer * * Copyright (C) 2014-2015 Emmanuel Keller / Jaeksoft * * http://www.open-search-server.com * * This file is part of OpenSearchServer. * * OpenSearchServer is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * OpenSearchServer is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with OpenSearchServer. * If not, see <http://www.gnu.org/licenses/>. **/ package com.jaeksoft.searchlib.crawler.mailbox.crawler; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Date; import java.util.List; import javax.activation.DataSource; import javax.mail.Address; import javax.mail.FetchProfile; import javax.mail.Flags.Flag; import javax.mail.Folder; import javax.mail.Message; import javax.mail.Message.RecipientType; import javax.mail.MessagingException; import javax.mail.Store; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; import org.apache.commons.mail.util.MimeMessageParser; import com.jaeksoft.searchlib.Logging; import com.jaeksoft.searchlib.SearchLibException; import com.jaeksoft.searchlib.crawler.FieldMapContext; import com.jaeksoft.searchlib.crawler.common.process.CrawlStatus; import com.jaeksoft.searchlib.crawler.mailbox.MailboxCrawlItem; import com.jaeksoft.searchlib.crawler.mailbox.MailboxCrawlThread; import com.jaeksoft.searchlib.crawler.mailbox.MailboxFieldEnum; import com.jaeksoft.searchlib.crawler.mailbox.MailboxProtocolEnum; import com.jaeksoft.searchlib.index.IndexDocument; import com.jaeksoft.searchlib.parser.Parser; import com.jaeksoft.searchlib.parser.ParserResultItem; import com.jaeksoft.searchlib.parser.ParserSelector; import com.jaeksoft.searchlib.util.IOUtils; import com.jaeksoft.searchlib.util.StringUtils; public abstract class MailboxAbstractCrawler { protected ParserSelector parserSelector; protected MailboxCrawlThread thread; protected MailboxProtocolEnum protocol; protected MailboxCrawlItem item; public void init(MailboxCrawlThread thread, MailboxProtocolEnum protocol, MailboxCrawlItem item) { this.thread = thread; this.protocol = protocol; this.item = item; FieldMapContext fieldMapContext = thread == null ? null : thread.getFieldMapContext(); this.parserSelector = fieldMapContext == null ? null : fieldMapContext.parserSelector; } protected abstract Store getStore() throws MessagingException; protected abstract void connect(Store store) throws MessagingException; public void read() throws MessagingException, IOException, SearchLibException { Store store = null; try { store = getStore(); connect(store); readFolder(store.getDefaultFolder()); } finally { if (store != null) store.close(); } } public String check() throws MessagingException, IOException, SearchLibException { Store store = null; StringWriter sw = null; PrintWriter pw = null; try { sw = new StringWriter(); pw = new PrintWriter(sw); pw.println(); store = getStore(); connect(store); checkFolder(store.getDefaultFolder(), pw); pw.println("OK"); return sw.toString(); } finally { if (store != null) store.close(); IOUtils.close(pw, sw); } } private void readMessagesFolder(Folder folder) throws MessagingException, IOException, SearchLibException { folder.open(Folder.READ_ONLY); String folderFullName = folder.getFullName(); try { int max = folder.getMessageCount(); int i = 0; final int buffer = item.getBufferSize(); while (i < max && !thread.isAborted()) { thread.setStatusInfo(CrawlStatus.CRAWL); int end = i + buffer; if (end > max) end = max; Message[] messages = folder.getMessages(i + 1, end); FetchProfile fp = new FetchProfile(); fp.add(FetchProfile.Item.ENVELOPE); folder.fetch(messages, fp); for (Message message : messages) { if (thread.isAborted()) break; i++; String messageId = getMessageId(folder, message); if (StringUtils.isEmpty(messageId)) continue; if (thread.isAlreadyIndexed(messageId)) { thread.incIgnored(); continue; } IndexDocument crawlIndexDocument = new IndexDocument(item.getLang()); IndexDocument parserIndexDocument = new IndexDocument(item.getLang()); crawlIndexDocument.addString(MailboxFieldEnum.folder.name(), folderFullName); try { readMessage(crawlIndexDocument, parserIndexDocument, folder, message, messageId); thread.addDocument(crawlIndexDocument, parserIndexDocument); } catch (Exception e) { Logging.warn(e); thread.incError(); } } } } finally { folder.close(false); } } protected abstract String getMessageId(Folder folder, Message message) throws MessagingException; protected void readFolder(Folder folder) throws MessagingException, IOException, SearchLibException { if (folder == null) return; if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) readMessagesFolder(folder); if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) readHoldsFolder(folder); } protected void checkFolder(Folder folder, PrintWriter pw) throws MessagingException, IOException, SearchLibException { if (folder == null) return; if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { folder.open(Folder.READ_ONLY); try { pw.print("Folder "); pw.print(folder.getName()); pw.print(": "); pw.print(folder.getMessageCount()); pw.println(" msgs(s)."); } finally { folder.close(false); } } if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { Folder[] folders = folder.list(); if (folders != null) for (Folder f : folders) checkFolder(f, pw); } } private void readHoldsFolder(Folder folder) throws MessagingException, IOException, SearchLibException { Folder[] folders = folder.list(); if (folders == null) return; for (Folder f : folders) readFolder(f); } private void putAddresses(IndexDocument document, Address[] addresses, String fieldEmail, String fieldPersonal) { if (addresses == null) return; for (Address address : addresses) { if (address == null) continue; if (!(address instanceof InternetAddress)) continue; InternetAddress ia = (InternetAddress) address; document.addString(fieldEmail, ia.getAddress()); document.addString(fieldPersonal, ia.getPersonal()); } } final public void readMessage(IndexDocument crawlIndexDocument, IndexDocument parserIndexDocument, Folder folder, Message message, String id) throws Exception { crawlIndexDocument.addString(MailboxFieldEnum.message_id.name(), id); crawlIndexDocument.addString(MailboxFieldEnum.message_number.name(), Integer.toString(message.getMessageNumber())); if (message instanceof MimeMessage) crawlIndexDocument.addString(MailboxFieldEnum.content_id.name(), ((MimeMessage) message).getContentID()); crawlIndexDocument.addString(MailboxFieldEnum.subject.name(), message.getSubject()); putAddresses(crawlIndexDocument, message.getFrom(), MailboxFieldEnum.from_address.name(), MailboxFieldEnum.from_personal.name()); putAddresses(crawlIndexDocument, message.getReplyTo(), MailboxFieldEnum.reply_to_address.name(), MailboxFieldEnum.reply_to_personal.name()); putAddresses(crawlIndexDocument, message.getRecipients(RecipientType.TO), MailboxFieldEnum.recipient_to_address.name(), MailboxFieldEnum.recipient_to_personal.name()); putAddresses(crawlIndexDocument, message.getRecipients(RecipientType.CC), MailboxFieldEnum.recipient_cc_address.name(), MailboxFieldEnum.recipient_cc_personal.name()); putAddresses(crawlIndexDocument, message.getRecipients(RecipientType.BCC), MailboxFieldEnum.recipient_bcc_address.name(), MailboxFieldEnum.recipient_bcc_personal.name()); Date dt = message.getSentDate(); if (dt != null) crawlIndexDocument.addString(MailboxFieldEnum.send_date.name(), dt.toString()); dt = message.getReceivedDate(); if (dt != null) crawlIndexDocument.addString(MailboxFieldEnum.received_date.name(), dt.toString()); if (message.isSet(Flag.ANSWERED)) crawlIndexDocument.addString(MailboxFieldEnum.flags.name(), "ANSWERED"); if (message.isSet(Flag.DELETED)) crawlIndexDocument.addString(MailboxFieldEnum.flags.name(), "DELETED"); if (message.isSet(Flag.DRAFT)) crawlIndexDocument.addString(MailboxFieldEnum.flags.name(), "DRAFT"); if (message.isSet(Flag.FLAGGED)) crawlIndexDocument.addString(MailboxFieldEnum.flags.name(), "FLAGGED"); if (message.isSet(Flag.SEEN)) crawlIndexDocument.addString(MailboxFieldEnum.flags.name(), "SEEN"); if (message instanceof MimeMessage) { MimeMessageParser mimeMessageParser = new MimeMessageParser((MimeMessage) message).parse(); crawlIndexDocument.addString(MailboxFieldEnum.html_content.name(), mimeMessageParser.getHtmlContent()); crawlIndexDocument.addString(MailboxFieldEnum.plain_content.name(), mimeMessageParser.getPlainContent()); for (DataSource dataSource : mimeMessageParser.getAttachmentList()) { crawlIndexDocument.addString(MailboxFieldEnum.email_attachment_name.name(), dataSource.getName()); crawlIndexDocument.addString(MailboxFieldEnum.email_attachment_type.name(), dataSource.getContentType()); if (parserSelector == null) continue; Parser attachParser = parserSelector.parseStream(null, dataSource.getName(), dataSource.getContentType(), null, dataSource.getInputStream(), null, null, null); if (attachParser == null) continue; List<ParserResultItem> parserResults = attachParser.getParserResults(); if (parserResults != null) for (ParserResultItem parserResult : parserResults) crawlIndexDocument.addFieldIndexDocument(MailboxFieldEnum.email_attachment_content.name(), parserResult.getParserDocument()); } } } }