List of usage examples for org.apache.lucene.index IndexWriter close
@Override public void close() throws IOException
From source file:com.khepry.frackhem.entities.Toxicities.java
License:Apache License
public void indexViaLucene(String textFilePath, String textColSeparator) throws IOException { String message;/* w w w. j av a 2 s. c om*/ message = "Start Indexing Toxicities via Lucene..."; if (outputToSystemOut) { System.out.println(message); } if (outputToMsgQueue) { progressMessageQueue.send(new MessageInput(message)); } File textFile = new File(textFilePath); if (textFile.exists()) { File indexFolder = new File(indexFolderPath); if (!indexFolder.exists()) { indexFolder.mkdir(); } else { deleteFolder(indexFolder); if (!indexFolder.exists()) { indexFolder.mkdir(); } } File taxonomyFolder = new File(taxonomyFolderPath); if (!taxonomyFolder.exists()) { taxonomyFolder.mkdir(); } else { deleteFolder(taxonomyFolder); if (!taxonomyFolder.exists()) { taxonomyFolder.mkdir(); } } if (indexFolder.exists() && taxonomyFolder.exists()) { List<String> colHeaders = new ArrayList<>(); Map<String, String> mapIndexFields = new LinkedHashMap<>(); Map<String, String> mapStatsFields = new LinkedHashMap<>(); String[] pieces; String[] tuples; pieces = indexFields.split(","); for (String indexField : pieces) { mapIndexFields.put(indexField, indexField); } pieces = statsFields.split(","); for (String statField : pieces) { tuples = statField.split(":"); mapStatsFields.put(tuples[0], tuples.length > 1 ? tuples[1] : tuples[0]); } SimpleFSDirectory indexDirectory = new SimpleFSDirectory(indexFolder); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_44); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_44, analyzer); IndexWriter indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); SimpleFSDirectory taxonomyDirectory = new SimpleFSDirectory(taxonomyFolder); TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxonomyDirectory, OpenMode.CREATE); FacetFields facetFields = new FacetFields(taxonomyWriter); List<CategoryPath> taxonomyCategories = new ArrayList<>(); String line; Integer rcdCount = 0; StringBuilder sb = new StringBuilder(); BufferedReader br = new BufferedReader(new FileReader(textFile)); while ((line = br.readLine()) != null) { rcdCount++; pieces = line.split(textColSeparator); if (rcdCount == 1) { for (String colHeader : pieces) { colHeaders.add(colHeader.trim()); } } else { if (pieces.length == colHeaders.size()) { sb.setLength(0); Document document = new Document(); for (int i = 0; i < pieces.length; i++) { Field field = new TextField(colHeaders.get(i), pieces[i].trim(), Store.YES); document.add(field); if (mapIndexFields.containsKey(colHeaders.get(i))) { if (!pieces[i].trim().equals("")) { sb.append(pieces[i].trim()); sb.append(" "); } } } Field field = new TextField("text", sb.toString().trim(), Store.NO); document.add(field); String toxCasEdfId = pieces[0].trim(); String toxChemical = pieces[1].trim(); // categorize recognized toxicities String toxRecognized = pieces[2].trim(); if (!toxRecognized.equals("")) { taxonomyCategories.add(new CategoryPath("toxRecognized", "CasEdfId", toxCasEdfId)); taxonomyCategories.add(new CategoryPath("toxRecognized", "Chemical", toxChemical.replace("/", "|"))); for (String value : toxRecognized.replace(" ", ",").split(",")) { if (!value.trim().equals("")) { taxonomyCategories .add(new CategoryPath("toxRecognized", "Toxicity", value)); } } } // categorize suspected toxicities String toxSuspected = pieces[3].trim(); if (!toxSuspected.equals("")) { taxonomyCategories.add(new CategoryPath("toxSuspected", "CasEdfId", toxCasEdfId)); taxonomyCategories.add(new CategoryPath("toxSuspected", "Chemical", toxChemical.replace("/", "|"))); for (String value : toxSuspected.replace(" ", ",").split(",")) { if (!value.trim().equals("")) { taxonomyCategories.add(new CategoryPath("toxSuspected", "Toxicity", value)); } } } // build up "stats" taxonomy categories for (String statsKey : mapStatsFields.keySet()) { if (mapIndexFields.containsKey(statsKey)) { String fieldValue = mapIndexFields.get(statsKey); if (!statsKey.trim().equals("") && !fieldValue.trim().equals("")) { taxonomyCategories .add(new CategoryPath("Toxicities", statsKey, fieldValue)); } } } if (taxonomyCategories.size() > 0) { facetFields.addFields(document, taxonomyCategories); // System.out.println("Taxonomies added: " + // taxonomyCategories.size()); } indexWriter.addDocument(document); if (progressInterval > 0 && rcdCount % progressInterval == 0) { message = "Records indexed: " + rcdCount; if (outputToSystemOut) { System.out.println(message); } if (outputToMsgQueue) { progressMessageQueue.send(new MessageInput(message)); } } taxonomyCategories.clear(); } } } br.close(); message = "Records indexed: " + rcdCount; if (outputToSystemOut) { System.out.println(message); } if (outputToMsgQueue) { progressMessageQueue.send(new MessageInput(message)); } sb.setLength(0); sb.trimToSize(); indexWriter.commit(); indexWriter.forceMerge(1); indexWriter.close(); taxonomyWriter.commit(); taxonomyWriter.close(); analyzer.close(); indexDirectory.close(); taxonomyDirectory.close(); } else { message = "Lucene Index Folder: " + indexFolder + " or Lucene Taxonomy folder: " + taxonomyFolder + " does not exist!"; if (outputToSystemErr) { System.err.println(message); } } message = "Ended Indexing Toxicities via Lucene!"; if (outputToSystemOut) { System.out.println(message); } if (outputToMsgQueue) { progressMessageQueue.send(new MessageInput(message)); } } }
From source file:com.knowgate.lucene.BugIndexer.java
License:Open Source License
/** * Add bug to index/* w ww . ja v a2 s.com*/ * @param oProps Properties * @param oCon JDCConnection * @param sWorkArea String * @param oBug Bug * @throws SQLException * @throws IOException * @throws ClassNotFoundException * @throws NoSuchFieldException * @throws IllegalAccessException * @throws InstantiationException * @throws NullPointerException * @throws NoSuchFieldException */ public static void addBug(Properties oProps, JDCConnection oCon, String sWorkArea, Bug oBug) throws SQLException, IOException, ClassNotFoundException, NoSuchFieldException, IllegalAccessException, InstantiationException, NullPointerException, NoSuchFieldException { String sDirectory = oProps.getProperty("luceneindex"); if (null == sDirectory) throw new NoSuchFieldException("Cannot find luceneindex property"); sDirectory = Gadgets.chomp(sDirectory, File.separator) + DB.k_bugs + File.separator + sWorkArea; File oDir = new File(sDirectory); if (!oDir.exists()) { FileSystem oFS = new FileSystem(); try { oFS.mkdirs(sDirectory); } catch (Exception e) { throw new IOException(e.getClass().getName() + " " + e.getMessage()); } } // fi Directory oFsDir = Indexer.openDirectory(sDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, Indexer.instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); addBug(oIWrt, oCon, sWorkArea, oBug); oIWrt.close(); oFsDir.close(); }
From source file:com.knowgate.lucene.ContactIndexer.java
License:Open Source License
public static void addOrReplaceContact(Properties oProps, String sGuid, String sWorkArea, JDCConnection oConn) throws ClassNotFoundException, IOException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InstantiationException, NullPointerException, SQLException { String sDirectory = oProps.getProperty("luceneindex"); if (DebugFile.trace) { DebugFile.writeln("Begin ContactIndexer.addOrReplaceContact([Properties]," + sGuid + "," + sWorkArea + ", [JDCConnection])"); DebugFile.incIdent();/*w w w. j a v a 2 s .c o m*/ } if (null == sDirectory) { if (DebugFile.trace) DebugFile.decIdent(); throw new NoSuchFieldException("Cannot find luceneindex property"); } sDirectory = Gadgets.chomp(sDirectory, File.separator) + "k_contacts" + File.separator + sWorkArea; if (DebugFile.trace) DebugFile.writeln("index directory is " + sDirectory); File oDir = new File(sDirectory); boolean bNewIndex = !oDir.exists(); if (oDir.exists()) { File[] aFiles = oDir.listFiles(); if (aFiles == null) { bNewIndex = true; } else if (aFiles.length == 0) { bNewIndex = true; } } if (bNewIndex) { Indexer.rebuild(oProps, "k_contacts", sWorkArea); } if (DebugFile.trace) DebugFile.writeln("IndexReader.open(" + sDirectory + ")"); Directory oFsDir = Indexer.openDirectory(sDirectory); IndexReader oIRdr = IndexReader.open(oFsDir); oIRdr.deleteDocuments(new Term("guid", sGuid)); oIRdr.close(); IndexWriter oIWrt = new IndexWriter(oFsDir, instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); addDocument(oIWrt, sGuid, sWorkArea, oConn); oIWrt.close(); oFsDir.close(); if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("End ContactIndexer.addOrReplaceContact()"); } }
From source file:com.knowgate.lucene.Crawler.java
License:Open Source License
/** * <p>Add contents to a Lucene Index * @param sBasePath Base Path for crawling * @param sFileFilter Perl5 Regular Expression filter for file names * @param sIndexDirectory Lucene index target directory * @param bRebuild <b>true</b> if index must be deleted and fully rebuild. * @throws IOException//from w ww .j av a 2 s .c o m * @throws FileNotFoundException If sBasePath direcory does not exist * @throws MalformedPatternException If sFileFilter is not a valid Perl5 regular expression pattern */ public void crawl(String sBasePath, String sFileFilter, String sIndexDirectory, boolean bRebuild) throws IOException, MalformedPatternException { if (DebugFile.trace) { DebugFile.writeln("Begin Crawler.crawl(" + sBasePath + "," + sFileFilter + "," + sIndexDirectory + ")"); DebugFile.incIdent(); } Directory oFsDir = Indexer.openDirectory(sIndexDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, new StopAnalyzer(Version.LUCENE_33), IndexWriter.MaxFieldLength.UNLIMITED); if (sBasePath.endsWith(sSeparator)) sBasePath = sBasePath.substring(0, sBasePath.length() - 1); crawlDir(oIWrt, sBasePath, sBasePath.length(), new RegExpFilter(sFileFilter)); oIWrt.optimize(); oIWrt.close(); oFsDir.close(); if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("End Crawler.crawl()"); } }
From source file:com.knowgate.lucene.Indexer.java
License:Open Source License
/** * Optimize a given index//from w w w. j av a2 s . c o m * @param oProps Properties Collection (typically loaded from hipergate.cnf) * containing luceneindex property and (optionally) analyzer * @param sTableName String Name of table to be indexed (currently only k_bugs, k_newsmsgs or k_mime_msgs are permitted) * @param sWorkArea GUID of WorkArea to be optimized * @throws NoSuchFieldException * @throws IllegalArgumentException * @throws ClassNotFoundException * @throws IOException * @throws InstantiationException * @throws IllegalAccessException */ public static void optimize(Properties oProps, String sTableName, String sWorkArea) throws NoSuchFieldException, IllegalArgumentException, ClassNotFoundException, FileNotFoundException, IOException, InstantiationException, IllegalAccessException { if (!allowedTable(sTableName)) throw new IllegalArgumentException("Table name must be k_bugs or k_newsmsgs or k_mime_msgs"); if (DebugFile.trace) { DebugFile.writeln("Begin Indexer.rebuild([Properties]" + sTableName); DebugFile.incIdent(); } String sDirectory = oProps.getProperty("luceneindex"); if (null == sDirectory) { if (DebugFile.trace) DebugFile.decIdent(); throw new NoSuchFieldException("Cannot find luceneindex property"); } sDirectory = Gadgets.chomp(sDirectory, File.separator) + sTableName.toLowerCase(); if (null != sWorkArea) sDirectory += File.separator + sWorkArea; if (DebugFile.trace) DebugFile.writeln("index directory is " + sDirectory); File oDir = new File(sDirectory); if (!oDir.exists()) { if (DebugFile.trace) DebugFile.decIdent(); throw new FileNotFoundException("Directory " + sDirectory + " does not exist"); } if (DebugFile.trace) DebugFile.writeln("new IndexWriter(...)"); Directory oFsDir = openDirectory(sDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); if (DebugFile.trace) DebugFile.writeln("IndexWriter.optimize()"); oIWrt.optimize(); if (DebugFile.trace) DebugFile.writeln("IndexWriter.close()"); oIWrt.close(); oFsDir.close(); if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("End Indexer.optimize()"); } }
From source file:com.knowgate.lucene.Indexer.java
License:Open Source License
/** * <p>Rebuild Full Text Index for a table restricting to a given WorkArea</p> * Indexed documents have the following fields:<br> * <table border=1 cellpadding=4>/*from w w w . j av a2 s . c om*/ * <tr><td><b>Field Name</b></td><td><b>Description</b></td><td><b>Indexed</b></td><td><b>Stored</b></td></tr> * <tr><td>workarea</td><td>GUID of WorkArea</td><td align=middle>Yes</td><td align=middle>Yes</td></tr> * <tr><td>container</td><td>Name of Container (NewsGroup, Project, etc)</td><td align=middle>Yes</td><td align=middle>Yes</td></tr> * <tr><td>guid</td><td>GUID for Retrieved Object</td><td align=middle>Yes</td><td align=middle>Yes</td></tr> * <tr><td>number</td><td>Object Ordinal Identifier</td><td align=middle>Yes</td><td align=middle>Yes</td></tr> * <tr><td>title</td><td>Title or Subject</td><td align=middle>Yes</td><td align=middle>Yes</td></tr> * <tr><td>author</td><td>Author</td><td align=middle>Yes</td><td align=middle>Yes</td></tr> * <tr><td>text</td><td>Document Text</td><td align=middle>Yes</td><td align=middle>No</td></tr> * <tr><td>abstract</td><td>First 80 characters of text</td><td align=middle>No</td><td align=middle>Yes</td></tr> * </table> * @param oProps Properties Collection (typically loaded from hipergate.cnf) containing:<br> * <b>driver</b> : Class name for JDBC driver<br> * <b>dburl</b> : Database Connection URL<br> * <b>dbuser</b> : Database User<br> * <b>dbpassword</b> : Database User Password<br> * <b>luceneindex</b> : Base path for Lucene index directories, * the rebuilded index will be stored at a subdirectory called as the table name.<br> * @param sTableName Name of table to be indexed (currently only k_bugs, k_newsmsgs or k_mime_msgs are permitted) * <b>analyzer</b> : org.apache.lucene.analysis.Analyzer subclass name * @param sWorkArea GUID of WorkArea to be rebuilt * @throws NoSuchFieldException If any of the requiered properties of oProps is not found * @throws ClassNotFoundException If JDBC driver or analyzer classes are not found * @throws SQLException * @throws IOException * @throws IllegalArgumentException * @throws IllegalAccessException * @throws InstantiationException */ public static void rebuild(Properties oProps, String sTableName, String sWorkArea) throws SQLException, IOException, ClassNotFoundException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InstantiationException { String sGuid, sThread, sContainer, sTitle, sAuthor, sComments, sText; Date dtCreated; BigDecimal dNumber; int iNumber, iSize; final BigDecimal dZero = new BigDecimal(0); // Check whether table name is any of the allowed ones if (!allowedTable(sTableName)) throw new IllegalArgumentException("Table name must be k_bugs or k_newsmsgs or k_mime_msgs"); if (DebugFile.trace) { DebugFile.writeln("Begin Indexer.rebuild([Properties]," + sTableName + "," + sWorkArea + ")"); DebugFile.incIdent(); } // Get physical base path to index files from luceneindex property String sDirectory = oProps.getProperty("luceneindex"); if (null == sDirectory) { if (DebugFile.trace) DebugFile.decIdent(); throw new NoSuchFieldException("Cannot find luceneindex property"); } // Append WorkArea and table name to luceneindex base path sDirectory = Gadgets.chomp(sDirectory, File.separator) + sTableName.toLowerCase(); if (null != sWorkArea) sDirectory += File.separator + sWorkArea; if (DebugFile.trace) DebugFile.writeln("index directory is " + sDirectory); if (null == oProps.getProperty("driver")) { if (DebugFile.trace) DebugFile.decIdent(); throw new NoSuchFieldException("Cannot find driver property"); } if (null == oProps.getProperty("dburl")) { if (DebugFile.trace) DebugFile.decIdent(); throw new NoSuchFieldException("Cannot find dburl property"); } if (DebugFile.trace) DebugFile.writeln("Class.forName(" + oProps.getProperty("driver") + ")"); Class oDriver = Class.forName(oProps.getProperty("driver")); if (DebugFile.trace) DebugFile.writeln("IndexReader.open(" + sDirectory + ")"); // ********************************************************************* // Delete every document from this table and WorkArea before re-indexing File oDir = new File(sDirectory); if (oDir.exists()) { File[] aFiles = oDir.listFiles(); if (null != aFiles) { if (aFiles.length > 0) { Directory oFsDir = openDirectory(sDirectory); IndexReader oReader = IndexReader.open(oFsDir); int iDeleted = oReader.deleteDocuments(new Term("workarea", sWorkArea)); oReader.close(); oFsDir.close(); } } } else { FileSystem oFS = new FileSystem(); try { oFS.mkdirs(sDirectory); } catch (Exception e) { throw new IOException(e.getClass().getName() + " " + e.getMessage()); } } // ********************************************************************* if (DebugFile.trace) DebugFile.writeln("new IndexWriter(" + sDirectory + ",[Analyzer], true)"); Directory oFsDir = openDirectory(sDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); if (DebugFile.trace) DebugFile.writeln("DriverManager.getConnection(" + oProps.getProperty("dburl") + ", ...)"); Connection oConn = DriverManager.getConnection(oProps.getProperty("dburl"), oProps.getProperty("dbuser"), oProps.getProperty("dbpassword")); oConn.setAutoCommit(true); Statement oStmt = oConn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ResultSet oRSet; if (sTableName.equalsIgnoreCase("k_bugs")) { if (DebugFile.trace) DebugFile.writeln( "Statement.executeQuery(SELECT p.gu_workarea,p.nm_project,b.gu_bug,b.tl_bug,b.dt_modified," + IfNull(oConn) + "(b.nm_reporter,'')," + IfNull(oConn) + "(b.tx_bug_brief,'')," + IfNull(oConn) + "(b.tx_comments,'') FROM k_bugs b, k_projects p WHERE b.gu_project=p.gu_project AND p.gu_owner='" + sWorkArea + "')"); oRSet = oStmt.executeQuery("SELECT p.gu_owner,p.nm_project,b.gu_bug,b.pg_bug,b.tl_bug,b.dt_modified," + IfNull(oConn) + "(b.nm_reporter,'')," + IfNull(oConn) + "(b.tx_comments,'')," + IfNull(oConn) + "(b.tx_bug_brief,'') FROM k_bugs b, k_projects p WHERE b.gu_project=p.gu_project AND p.gu_owner='" + sWorkArea + "'"); while (oRSet.next()) { sWorkArea = oRSet.getString(1); sContainer = oRSet.getString(2); sGuid = oRSet.getString(3); iNumber = oRSet.getInt(4); sTitle = oRSet.getString(5); dtCreated = oRSet.getDate(6); sAuthor = oRSet.getString(7); sComments = oRSet.getString(8); if (null == sComments) sComments = ""; sText = oRSet.getString(9); if (null == sText) sText = ""; BugIndexer.addBug(oIWrt, sGuid, iNumber, sWorkArea, sContainer, sTitle, sAuthor, dtCreated, sComments, sText); } // wend oRSet.close(); } else if (sTableName.equalsIgnoreCase("k_newsmsgs")) { if (DebugFile.trace) DebugFile.writeln( "Statement.executeQuery(SELECT g.gu_workarea,c.nm_category,m.gu_msg,m.tx_subject,m.dt_published," + IfNull(oConn) + "(b.nm_author,'')," + IfNull(oConn) + "(b.tx_msg,'') FROM k_newsmsgs m, k_categories c, k_newsgroups g, k_x_cat_objs x WHERE m.id_status=0 AND m.gu_msg=x.gu_object AND x.gu_category=g.gu_newsgrp AND c.gu_category=g.gu_newsgrp AND g.gu_workarea='" + sWorkArea + "')"); oRSet = oStmt.executeQuery( "SELECT g.gu_workarea,c.nm_category,m.gu_msg,m.gu_thread_msg,m.tx_subject,m.dt_published," + IfNull(oConn) + "(m.nm_author,'')," + IfNull(oConn) + "(m.tx_msg,'') FROM k_newsmsgs m, k_categories c, k_newsgroups g, k_x_cat_objs x WHERE m.id_status=0 AND m.gu_msg=x.gu_object AND x.gu_category=g.gu_newsgrp AND c.gu_category=g.gu_newsgrp AND g.gu_workarea='" + sWorkArea + "'"); while (oRSet.next()) { sWorkArea = oRSet.getString(1); sContainer = oRSet.getString(2); sGuid = oRSet.getString(3); sThread = oRSet.getString(4); sTitle = oRSet.getString(5); dtCreated = oRSet.getDate(6); sAuthor = oRSet.getString(7); sText = oRSet.getString(8); NewsMessageIndexer.addNewsMessage(oIWrt, sGuid, sThread, sWorkArea, sContainer, sTitle, sAuthor, dtCreated, sText); } // wend oRSet.close(); } // Inicio I2E 2009-12-23 else if (sTableName.equalsIgnoreCase("k_contacts")) { Map<String, ContactRecord> contacts = new HashMap<String, ContactRecord>(); String consultas[] = new String[6]; consultas[0] = "SELECT c.gu_contact, c.gu_workarea, c.tx_name, c.tx_surname, csc.nm_scourse, csc.lv_scourse FROM k_contacts c, k_contact_short_courses csc WHERE c.gu_workarea='" + sWorkArea + "' AND csc.gu_contact = c.gu_contact"; consultas[1] = "SELECT c.gu_contact, c.gu_workarea, c.tx_name, c.tx_surname, ccsl.tr_es,ccsl2.tr_es FROM k_contacts c, k_contact_computer_science ccc, k_contact_computer_science_lookup ccsl, k_contact_computer_science_lookup ccsl2 WHERE c.gu_workarea='" + sWorkArea + "' AND ccc.gu_contact = c.gu_contact AND ccc.nm_skill = ccsl.vl_lookup AND ccc.lv_skill = ccsl2.vl_lookup"; consultas[2] = "SELECT c.gu_contact, c.gu_workarea, c.tx_name, c.tx_surname, ccsl.tr_en,ccsl2.tr_en FROM k_contacts c, k_contact_computer_science ccc, k_contact_computer_science_lookup ccsl, k_contact_computer_science_lookup ccsl2 WHERE c.gu_workarea='" + sWorkArea + "' AND ccc.gu_contact = c.gu_contact AND ccc.nm_skill = ccsl.vl_lookup AND ccc.lv_skill = ccsl2.vl_lookup"; consultas[3] = "SELECT c.gu_contact, c.gu_workarea, c.tx_name, c.tx_surname, ed.nm_degree,'' as level FROM k_contacts c,k_contact_education ce,k_education_degree ed WHERE c.gu_workarea='" + sWorkArea + "' AND ce.gu_contact = c.gu_contact AND ce.gu_degree= ed.gu_degree"; consultas[4] = "SELECT c.gu_contact, c.gu_workarea, c.tx_name, c.tx_surname, ll.tr_lang_es,cll.tr_es FROM k_contacts c, k_contact_languages cl, k_lu_languages ll,k_contact_languages_lookup cll WHERE c.gu_workarea='" + sWorkArea + "' AND c.gu_contact = cl.gu_contact AND cl.id_language = ll.id_language AND cl.lv_language_degree = cll.vl_lookup"; consultas[5] = "SELECT c.gu_contact, c.gu_workarea, c.tx_name, c.tx_surname, ll.tr_lang_en,cll.tr_en FROM k_contacts c, k_contact_languages cl, k_lu_languages ll,k_contact_languages_lookup cll WHERE c.gu_workarea='" + sWorkArea + "' AND c.gu_contact = cl.gu_contact AND cl.id_language = ll.id_language AND cl.lv_language_degree = cll.vl_lookup"; for (int i = 0; i < consultas.length; i++) { if (DebugFile.trace) DebugFile.writeln("Statement.executeQuery(" + consultas[i] + ")"); oRSet = oStmt.executeQuery(consultas[i]); while (oRSet.next()) { sGuid = oRSet.getString(1); sWorkArea = oRSet.getString(2); String sName = oRSet.getString(3); String sSurname = oRSet.getString(4); String sValue = oRSet.getString(5); String sLevel = oRSet.getString(6); if (sLevel == null) sLevel = ""; ContactRecord contact = contacts.get(sGuid); if (contact == null) { contact = new ContactRecord(null, sName + " " + sSurname, sWorkArea, sGuid); contacts.put(sGuid, contact); } contact.addValue(sValue, sLevel); //ContactIndexer.addDocument(oIWrt, sGuid, sWorkArea, sName, sSurname, ContactRecord.COURSE, sValue, sLevel,null); } oRSet.close(); } ContactRecord arrayContactos[] = contacts.values().toArray(new ContactRecord[contacts.size()]); for (int i = 0; i < arrayContactos.length; i++) { ContactIndexer.addDocument(oIWrt, arrayContactos[i]); } } //Fin i2E else if (sTableName.equalsIgnoreCase("k_mime_msgs")) { LinkedList oIndexedGuids = new LinkedList(); PreparedStatement oRecp = oConn.prepareStatement( "SELECT tx_personal,tx_email FROM k_inet_addrs WHERE tp_recipient<>'to' AND gu_mimemsg=?", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); if (DebugFile.trace) DebugFile.writeln( "Statement.executeQuery(SELECT g.gu_workarea,c.nm_category,m.gu_mimemsg,m.tx_subject,m.nm_from,m.tx_mail_from,m.pg_mimemsg,m.de_mimemsg,m.dt_sent,m.len_mimemsg,m.by_content FROM k_mime_msgs m, k_categories c WHERE m.bo_deleted<>0 AND m.bo_draft<>0 AND m.gu_category=c.gu_category AND m.gu_workarea='" + sWorkArea + "')"); oRSet = oStmt.executeQuery( "SELECT g.gu_workarea,c.nm_category,m.gu_mimemsg,m.tx_subject,m.nm_from,m.tx_mail_from,m.pg_mimemsg,m.de_mimemsg,m.dt_sent,m.len_mimemsg,m.by_content FROM k_mime_msgs m, k_categories c WHERE m.bo_deleted<>0 AND m.bo_draft<>0 AND m.gu_category=c.gu_category AND m.gu_workarea='" + sWorkArea + "'"); while (oRSet.next()) { sWorkArea = oRSet.getString(1); sContainer = oRSet.getString(2); sGuid = oRSet.getString(3); sTitle = oRSet.getString(4); sAuthor = oRSet.getString(5); if (oRSet.wasNull()) sAuthor = ""; sAuthor += " " + oRSet.getString(6); dNumber = oRSet.getBigDecimal(7); if (oRSet.wasNull()) dNumber = dZero; sComments = oRSet.getString(8); dtCreated = oRSet.getDate(9); iSize = oRSet.getInt(10); if (DebugFile.trace) DebugFile.writeln("Indexing message " + sGuid + " - " + sTitle); InputStream oStrm = oRSet.getBinaryStream(11); String sRecipients = ""; oRecp.setString(1, sGuid); ResultSet oRecs = oRecp.executeQuery(); while (oRecs.next()) { sRecipients += oRecs.getString(1) + " " + oRecs.getString(2) + " "; } // wend oRecs.close(); MailIndexer.addMail(oIWrt, sGuid, dNumber, sWorkArea, sContainer, sTitle, sAuthor, sRecipients, dtCreated, sComments, oStrm, iSize); oIndexedGuids.add(sGuid); } // wend oRSet.close(); oRecp.close(); PreparedStatement oUpdt = oConn .prepareStatement("UPDATE k_mime_msgs SET bo_indexed=1 WHERE gu_mimemsg=?"); ListIterator oIter = oIndexedGuids.listIterator(); while (oIter.hasNext()) { oUpdt.setObject(1, oIter.next(), java.sql.Types.CHAR); oUpdt.executeUpdate(); } // wend oUpdt.close(); } // fi oStmt.close(); oConn.close(); if (DebugFile.trace) DebugFile.writeln("IndexWriter.optimize()"); oIWrt.optimize(); if (DebugFile.trace) DebugFile.writeln("IndexWriter.close()"); oIWrt.close(); oFsDir.close(); if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("End Indexer.rebuild()"); } }
From source file:com.knowgate.lucene.Indexer.java
License:Open Source License
public static void add(String sTableName, String sDirectory, String sAnalyzer, Map oKeywords, Map oTexts, Map oUnStored) throws ClassNotFoundException, IOException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InstantiationException, NullPointerException { if (!allowedTable(sTableName)) throw new IllegalArgumentException("Table name must be k_bugs or k_newsmsgs or k_mime_msgs"); if (null == sDirectory) throw new NoSuchFieldException("Cannot find luceneindex property"); File oDir = new File(sDirectory); if (!oDir.exists()) { FileSystem oFS = new FileSystem(); try {/*from w w w. j a v a 2 s . c o m*/ oFS.mkdirs(sDirectory); } catch (Exception e) { throw new IOException(e.getClass().getName() + " " + e.getMessage()); } } Class oAnalyzer = Class.forName((sAnalyzer == null) ? DEFAULT_ANALYZER : sAnalyzer); NIOFSDirectory oFsDir = new NIOFSDirectory(new File(sDirectory)); IndexWriter oIWrt = new IndexWriter(oFsDir, (Analyzer) oAnalyzer.newInstance(), IndexWriter.MaxFieldLength.LIMITED); add(oIWrt, oKeywords, oTexts, oUnStored); oIWrt.close(); oFsDir.close(); }
From source file:com.knowgate.lucene.Indexer.java
License:Open Source License
/** * Add a document to the index/*from w w w . j a v a2s. c om*/ * @param sTableName k_bugs, k_newsmsgs or k_mime_msgs * @param oProps Properties Collection containing luceneindex directory * @param sWorkArea WorkArea for document * @param sContainer GUID of Category or NewsGroup to which documento belongs * @param sGUID Document GUID * @param iNumber Document number (optional, may be zero) * @param sTitle Document Title (optional, may be <b>null</b>) * @param sText Document text (optional, may be <b>null</b>) * @param sAuthor Document author (optional, may be <b>null</b>) * @param sAbstract Document abstract (optional, may be <b>null</b>) * @param sComments Document comments (optional, may be <b>null</b>) * @throws ClassNotFoundException * @throws IOException * @throws IllegalArgumentException If sTableName is not one of { k_bugs, k_newsmsgs, k_mime_msgs } * @throws NoSuchFieldException If luceneindex property is not found at oProps * @throws IllegalAccessException * @throws InstantiationException * @throws NullPointerException * @deprecated Use add method from Indexer subclasses instead */ public static void add(String sTableName, Properties oProps, String sGUID, int iNumber, String sWorkArea, String sContainer, String sTitle, String sText, String sAuthor, String sAbstract, String sComments) throws ClassNotFoundException, IOException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InstantiationException, NullPointerException { if (null == sGUID) throw new NullPointerException("Document GUID may not be null"); if (!sTableName.equalsIgnoreCase("k_bugs") && !sTableName.equalsIgnoreCase("k_newsmsgs") && !sTableName.equalsIgnoreCase("k_mime_msgs")) throw new IllegalArgumentException("Table name must be k_bugs or k_newsmsgs or k_mime_msgs"); String sDirectory = oProps.getProperty("luceneindex"); if (null == sDirectory) throw new NoSuchFieldException("Cannot find luceneindex property"); sDirectory = Gadgets.chomp(sDirectory, File.separator) + sTableName.toLowerCase() + File.separator + sWorkArea; File oDir = new File(sDirectory); if (!oDir.exists()) { FileSystem oFS = new FileSystem(); try { oFS.mkdirs(sDirectory); } catch (Exception e) { throw new IOException(e.getClass().getName() + " " + e.getMessage()); } } HashMap oKeys = new HashMap(11); oKeys.put("workarea", sWorkArea == null ? "" : sWorkArea); oKeys.put("container", sContainer == null ? "" : sContainer); oKeys.put("guid", sGUID); oKeys.put("number", String.valueOf(iNumber)); HashMap oTexts = new HashMap(11); oTexts.put("title", sTitle == null ? "" : sTitle); oTexts.put("author", sAuthor == null ? "" : sAuthor); oTexts.put("abstract", sAbstract == null ? "" : Gadgets.left(sAbstract, 80)); HashMap oUnstor = new HashMap(11); oUnstor.put("comments", sComments == null ? "" : sComments); oUnstor.put("text", sText == null ? "" : sText); Directory oFsDir = openDirectory(sDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); add(oIWrt, oKeys, oTexts, oUnstor); oIWrt.close(); oFsDir.close(); }
From source file:com.knowgate.lucene.MailIndexer.java
License:Open Source License
/** * <p>Re-build full text index for a given mail folder</p> * All previously indexed messages for given folder are removed from index and written back * @param oProps Properties containing: luceneindex, driver, dburl, dbuser, dbpassword * @param sWorkArea String GUID of WorkArea to which folder belongs * @param sFolder String Folder name as in field nm_category of table k_categories * @throws SQLException/*from w w w .j a va2s . co m*/ * @throws IOException * @throws ClassNotFoundException * @throws IllegalArgumentException * @throws NoSuchFieldException * @throws IllegalAccessException * @throws InstantiationException */ public static void rebuildFolder(Properties oProps, String sWorkArea, String sFolder) throws SQLException, IOException, ClassNotFoundException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InstantiationException { String sGuid, sContainer, sTitle, sAuthor, sComments; Date dtCreated; BigDecimal dNumber; int iSize; final BigDecimal dZero = new BigDecimal(0); if (DebugFile.trace) { DebugFile.writeln("Begin MailIndexer.rebuildFolder([Properties]" + sWorkArea + "," + sFolder + ")"); DebugFile.incIdent(); } // Get physical base path to index files from luceneindex property String sDirectory = oProps.getProperty("luceneindex"); if (null == sDirectory) throw new NoSuchFieldException("Cannot find luceneindex property"); // Append WorkArea and table name to luceneindex base path sDirectory = Gadgets.chomp(sDirectory, File.separator) + "k_mime_msgs"; if (null != sWorkArea) sDirectory += File.separator + sWorkArea; if (DebugFile.trace) DebugFile.writeln("index directory is " + sDirectory); if (null == oProps.getProperty("driver")) throw new NoSuchFieldException("Cannot find driver property"); if (null == oProps.getProperty("dburl")) throw new NoSuchFieldException("Cannot find dburl property"); if (DebugFile.trace) DebugFile.writeln("Class.forName(" + oProps.getProperty("driver") + ")"); Class oDriver = Class.forName(oProps.getProperty("driver")); if (DebugFile.trace) DebugFile.writeln("IndexReader.open(" + sDirectory + ")"); // ********************************************************************* // Delete every document from this folder before re-indexing File oDir = new File(sDirectory); if (oDir.exists()) { File[] aSegments = oDir.listFiles(); if (null != aSegments) { if (aSegments.length > 0) { Directory oRdDir = Indexer.openDirectory(sDirectory); IndexReader oReader = IndexReader.open(oRdDir); int iDeleted = oReader.deleteDocuments(new Term("container", sFolder)); oReader.close(); oRdDir.close(); } // fi } // fi } else { FileSystem oFS = new FileSystem(); try { oFS.mkdirs(sDirectory); } catch (Exception e) { throw new IOException(e.getClass().getName() + " " + e.getMessage()); } } // fi // ********************************************************************* if (DebugFile.trace) DebugFile.writeln("new IndexWriter(" + sDirectory + ",[Analyzer], true)"); Directory oFsDir = Indexer.openDirectory(sDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, Indexer.instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); if (DebugFile.trace) DebugFile.writeln("DriverManager.getConnection(" + oProps.getProperty("dburl") + ", ...)"); Connection oConn = DriverManager.getConnection(oProps.getProperty("dburl"), oProps.getProperty("dbuser"), oProps.getProperty("dbpassword")); Statement oStmt = oConn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); ResultSet oRSet; PreparedStatement oRecp = oConn.prepareStatement( "SELECT tx_personal,tx_email FROM k_inet_addrs WHERE tp_recipient<>'to' AND gu_mimemsg=?", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); if (DebugFile.trace) DebugFile.writeln( "Statement.executeQuery(SELECT m.gu_workarea,c.nm_category,m.gu_mimemsg,m.tx_subject,m.nm_from,m.tx_email_from,m.pg_message,m.de_mimemsg,m.dt_sent,m.len_mimemsg,m.by_content FROM k_mime_msgs m, k_categories c WHERE m.bo_deleted=0 AND m.bo_draft=0 AND m.gu_category=c.gu_category AND m.gu_workarea='" + sWorkArea + "' AND c.nm_category='" + sFolder + "')"); oRSet = oStmt.executeQuery( "SELECT m.gu_workarea,c.nm_category,m.gu_mimemsg,m.tx_subject,m.nm_from,m.tx_email_from,m.pg_message,m.de_mimemsg,m.dt_sent,m.len_mimemsg,m.by_content FROM k_mime_msgs m, k_categories c WHERE m.bo_deleted=0 AND m.bo_draft=0 AND m.gu_category=c.gu_category AND m.gu_workarea='" + sWorkArea + "' AND c.nm_category='" + sFolder + "'"); int nIndexed = 0; while (oRSet.next()) { sWorkArea = oRSet.getString(1); sContainer = oRSet.getString(2); sGuid = oRSet.getString(3); sTitle = oRSet.getString(4); sAuthor = oRSet.getString(5); if (oRSet.wasNull()) sAuthor = ""; sAuthor += " " + oRSet.getString(6); dNumber = oRSet.getBigDecimal(7); if (oRSet.wasNull()) dNumber = dZero; sComments = oRSet.getString(8); if (oRSet.wasNull()) sComments = ""; dtCreated = oRSet.getDate(9); iSize = oRSet.getInt(10); if (DebugFile.trace) DebugFile.writeln("Indexing message " + sGuid + " - " + sTitle); InputStream oStrm = oRSet.getBinaryStream(11); String sRecipients = ""; oRecp.setString(1, sGuid); ResultSet oRecs = oRecp.executeQuery(); while (oRecs.next()) { String sTxPersonal = oRecs.getString(1); if (oRecs.wasNull()) sRecipients += oRecs.getString(2) + " "; else sRecipients += oRecs.getString(1) + " " + oRecs.getString(2) + " "; } // wend oRecs.close(); MailIndexer.addMail(oIWrt, sGuid, dNumber, sWorkArea, sContainer, sTitle, sAuthor, sRecipients, dtCreated, sComments, oStrm, iSize); nIndexed++; } // wend if (DebugFile.trace) { DebugFile.writeln(String.valueOf(nIndexed) + " messages indexed"); } oRSet.close(); oRecp.close(); if (DebugFile.trace) { DebugFile.writeln("Statement.executeUpdate(UPDATE k_mime_msgs SET bo_indexed=1 WHERE gu_workarea='" + sWorkArea + "' AND gu_category IN (SELECT gu_category FROM k_categories WHERE nm_category='" + sFolder + "'))"); } oStmt.executeUpdate("UPDATE k_mime_msgs SET bo_indexed=1 WHERE gu_workarea='" + sWorkArea + "' AND gu_category IN (SELECT gu_category FROM k_categories WHERE nm_category='" + sFolder + "')"); oStmt.close(); oConn.close(); if (DebugFile.trace) DebugFile.writeln("IndexWriter.optimize()"); oIWrt.optimize(); if (DebugFile.trace) DebugFile.writeln("IndexWriter.close()"); oIWrt.close(); oFsDir.close(); if (DebugFile.trace) { DebugFile.decIdent(); DebugFile.writeln("End MailIndexer.rebuildFolder()"); } }
From source file:com.knowgate.lucene.NewsMessageIndexer.java
License:Open Source License
public static void addNewsMessage(Properties oProps, String sGuid, String sThread, String sWorkArea, String sContainer, String sTitle, String sAuthor, Date dtCreated, String sText) throws ClassNotFoundException, IOException, IllegalArgumentException, NoSuchFieldException, IllegalAccessException, InstantiationException, NullPointerException { String sDirectory = oProps.getProperty("luceneindex"); if (null == sDirectory) { if (DebugFile.trace) DebugFile.decIdent();//w w w .j a v a2 s . c om throw new NoSuchFieldException("Cannot find luceneindex property"); } sDirectory = Gadgets.chomp(sDirectory, File.separator) + "k_newsmsgs" + File.separator + sWorkArea; if (DebugFile.trace) DebugFile.writeln("index directory is " + sDirectory); Directory oFsDir = Indexer.openDirectory(sDirectory); IndexWriter oIWrt = new IndexWriter(oFsDir, Indexer.instantiateAnalyzer(oProps), IndexWriter.MaxFieldLength.LIMITED); addNewsMessage(oIWrt, sGuid, sThread, sWorkArea, sContainer, sTitle, sAuthor, dtCreated, sText); oIWrt.close(); oFsDir.close(); }