List of usage examples for org.apache.lucene.index IndexWriter addIndexes
public long addIndexes(CodecReader... readers) throws IOException
From source file:cn.hbu.cs.esearch.index.BaseSearchIndex.java
License:Apache License
public void loadFromIndex(BaseSearchIndex<R> index) throws IOException { LongSet delDocs = null;/*from w ww .j a v a 2 s . c o m*/ // delete docs in disk index first synchronized (this) { if (_delDocs != null && _delDocs.size() > 0) { delDocs = _delDocs; clearDeletes(); } } deleteDocs(delDocs); // open readOnly ram index reader EsearchMultiReader<R> reader = index.openIndexReader(); if (reader == null) { return; } // merge the readOnly ram index with the disk index IndexWriter writer = null; try { writer = openIndexWriter(null, null); writer.addIndexes(reader.directory()); writer.maybeMerge(); } finally { closeIndexWriter(); } }
From source file:com.duroty.service.BookmarkOptimizerThread.java
License:Open Source License
/** * DOCUMENT ME!/*ww w . jav a 2s .c o m*/ * * @param childs DOCUMENT ME! * * @throws Exception DOCUMENT ME! */ private void flush(File[] childs) throws Exception { if ((childs == null) || (childs.length == 0)) { return; } try { Thread.sleep(100); } catch (Exception e) { } File optimized = new File(optimizedPath); boolean create = false; IndexWriter writer = null; try { if (!IndexReader.indexExists(optimized)) { optimized.mkdirs(); create = true; } synchronized (this) { if (IndexReader.isLocked(optimizedPath)) { return; } else { Directory dir = FSDirectory.getDirectory(new File(optimizedPath), create); writer = new IndexWriter(dir, analyzer, create); } for (int i = 0; i < childs.length; i++) { boolean lock = true; File child = childs[i]; File[] faux = child.listFiles(); for (int j = 0; j < faux.length; j++) { if (faux[j].getName().equals("is.unlock")) { faux[j].delete(); lock = false; break; } } if (!lock) { Directory[] aux = new Directory[1]; aux[0] = FSDirectory.getDirectory(child, false); IndexSearcher searcher = null; try { searcher = new IndexSearcher(aux[0]); Document doc = searcher.doc(0); if (doc != null) { BookmarkIndexer.createSpell(userPath + SPELL, Field_title, doc); BookmarkIndexer.createSpell(userPath + SPELL, Field_keywords, doc); BookmarkIndexer.createSpell(userPath + SPELL, Field_contents, doc); BookmarkIndexer.createSpell(userPath + SPELL, Field_comments, doc); } } catch (Exception ex) { if ((ex != null) && !(ex instanceof NullPointerException)) { DLog.log(DLog.INFO, this.getClass(), ex); } } finally { if (searcher != null) { try { searcher.close(); } catch (Exception e) { } } } writer.addIndexes(aux); writer.optimize(); for (int j = 0; j < faux.length; j++) { faux[j].delete(); } child.delete(); } } writer.close(); writer = null; } } finally { if (writer != null) { try { writer.close(); } catch (Exception e) { } } } }
From source file:com.duroty.service.MailOptimizerThread.java
License:Open Source License
/** * DOCUMENT ME!/* w w w . j a va 2s .c om*/ * * @param childs DOCUMENT ME! * * @throws Exception DOCUMENT ME! */ private void flush(File[] childs) throws Exception { if ((childs == null) || (childs.length == 0)) { return; } try { Thread.sleep(500); } catch (Exception e) { } File optimized = new File(optimizedPath); boolean create = false; IndexWriter writer = null; try { if (!IndexReader.indexExists(optimized)) { optimized.mkdirs(); create = true; } synchronized (this) { if (IndexReader.isLocked(optimizedPath)) { return; } else { Directory dir = FSDirectory.getDirectory(new File(optimizedPath), create); writer = new IndexWriter(dir, analyzer, create); } for (int i = 0; i < childs.length; i++) { boolean lock = true; File child = childs[i]; File[] faux = child.listFiles(); for (int j = 0; j < faux.length; j++) { if (faux[j].getName().equals("is.unlock")) { faux[j].delete(); lock = false; break; } } if (!lock) { Directory[] aux = new Directory[1]; aux[0] = FSDirectory.getDirectory(child, false); IndexSearcher searcher = null; try { searcher = new IndexSearcher(aux[0]); Document doc = searcher.doc(0); if (doc != null) { MailIndexer.createSpell(userPath + SPELL, Field_from, doc); MailIndexer.createSpell(userPath + SPELL, Field_to, doc); MailIndexer.createSpell(userPath + SPELL, Field_subject, doc); MailIndexer.createSpell(userPath + SPELL, Field_body, doc); } } catch (Exception ex) { if ((ex != null) && !(ex instanceof NullPointerException)) { DLog.log(DLog.INFO, this.getClass(), ex); } } finally { if (searcher != null) { try { searcher.close(); } catch (Exception e) { } } } writer.addIndexes(aux); writer.optimize(); for (int j = 0; j < faux.length; j++) { faux[j].delete(); } child.delete(); } } writer.close(); writer = null; } } finally { if (writer != null) { try { writer.close(); } catch (Exception e) { } } } }
From source file:com.flaptor.hounder.indexer.util.MergeIndexes.java
License:Apache License
public static void main(String args[]) { if (args.length < 3) { System.err.println("Usage: MergeIndexes outputIndex index1 index2 [... indexN]"); System.exit(-1);/*from w w w.j a v a2s .com*/ } try { IndexWriter iw = new IndexWriter(args[0], new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); iw.setMaxBufferedDocs(1000); IndexReader readers[] = new IndexReader[args.length - 1]; for (int i = 0; i < args.length - 1; i++) { readers[i] = IndexReader.open(args[i + 1]); } iw.addIndexes(readers); iw.optimize(); iw.close(); } catch (IOException e) { System.err.println(e); } }
From source file:com.flaptor.hounder.util.Idx.java
License:Apache License
public static void main(String arg[]) throws Exception { check(arg.length > 1, null);/* w w w .ja va 2 s. c o m*/ String cmd = arg[0]; File idx = new File(arg[1]); if ("list".equals(cmd)) { int num = (arg.length > 2) ? Integer.parseInt(arg[2]) : -1; check(idx.exists(), "Index dir not found"); IndexReader reader = IndexReader.open(idx); int docs = reader.numDocs(); int max = reader.maxDoc(); System.err.println("Index contains " + docs + " documents plus " + (max - docs) + " deleted."); if (num > -1) { if (num == 0) num = docs; for (int i = 0; i < max && i < num; i++) { System.out.println("----------------------------------------"); if (!reader.isDeleted(i)) { Document doc = reader.document(i); List flds = doc.getFields(); Iterator iter = flds.iterator(); while (iter.hasNext()) { Field fld = (Field) iter.next(); String attr = (fld.isIndexed() ? ",i" : "") + (fld.isStored() ? ",s" : "") + (fld.isTokenized() ? ",t" : ""); System.out.println(fld.name() + attr + ": " + fld.stringValue()); } } } reader.close(); System.out.println(); } } else if ("search".equals(cmd)) { check(idx.exists(), "Index dir not found"); check(arg.length > 3, "Not enough arguments"); String field = arg[2]; String value = arg[3]; IndexSearcher searcher = new IndexSearcher(IndexReader.open(idx)); ScorelessHitCollector collector = new HashSetScorelessHitCollector(); searcher.search(new TermQuery(new Term(field, value)), collector); Set<Integer> docIds = collector.getMatchingDocuments(); System.out.println("\nNumber of hits: " + docIds.size() + "\n"); for (Integer docId : docIds) { Document doc = searcher.doc(docId); List flds = doc.getFields(); Iterator iter = flds.iterator(); while (iter.hasNext()) { Field fld = (Field) iter.next(); System.out.println(fld.name() + ": " + fld.stringValue()); } } searcher.close(); System.out.println(); } else if ("delete".equals(cmd)) { check(idx.exists(), "Index dir not found"); check(arg.length > 3, "Not enough arguments"); String field = arg[2]; String value = arg[3]; IndexReader reader = IndexReader.open(idx); reader.deleteDocuments(new Term(field, value)); reader.close(); } else if ("optimize".equals(cmd)) { IndexWriter writer = new IndexWriter(idx, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); writer.optimize(); writer.close(); } else if ("merge".equals(cmd)) { check(arg.length == 3, "not enough parameters"); File idx2 = new File(arg[2]); check(idx.exists(), "Index dir 1 not found"); check(idx2.exists(), "Index dir 2 not found"); IndexReader reader = IndexReader.open(idx2); IndexWriter writer = new IndexWriter(idx, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); writer.addIndexes(new IndexReader[] { reader }); writer.close(); reader.close(); } else if ("term-count".equals(cmd)) { check(arg.length == 3, "not enough parameters"); check(idx.exists(), "Index dir not found"); IndexReader reader = IndexReader.open(idx); String field = arg[2]; int count = 0; TermEnum terms = reader.terms(); while (terms.next()) { Term term = terms.term(); if (term.field().equals(field)) count++; } terms.close(); reader.close(); System.out.println("Found " + count + " different values for field " + field); } else if ("hit-count".equals(cmd)) { check(arg.length > 3, "Not enough arguments"); check(idx.exists(), "Index dir not found"); String field = arg[2]; String value = arg[3]; IndexSearcher searcher = new IndexSearcher(IndexReader.open(idx)); CountingHitCollector collector = new CountingHitCollector(); searcher.search(new TermQuery(new Term(field, value)), collector); System.out.println("\nNumber of hits: " + collector.getDocCount() + "\n"); searcher.close(); } else if ("uncompound".equals(cmd)) { IndexWriter writer = new IndexWriter(idx, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); writer.setUseCompoundFile(false); writer.optimize(); writer.close(); } else if ("compound".equals(cmd)) { IndexWriter writer = new IndexWriter(idx, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); writer.setUseCompoundFile(true); writer.optimize(); writer.close(); } else if ("terms".equals(cmd)) { check(arg.length == 3, "not enough parameters"); check(idx.exists(), "Index dir not found"); String field = arg[2]; IndexReader reader = IndexReader.open(idx); TermEnum terms = reader.terms(); while (terms.next()) { Term t = terms.term(); if (t.field().equals(field)) { System.out.println(t.text()); } } } }
From source file:com.globalsight.ling.lucene.Index.java
License:Apache License
/** * Ends the batch re-creation of an index by clearing out the old * index files, writing the new in-memory index to disk, and * setting the index state to STATE_OPENED. * * @see #batchOpen()/*from w w w. j a va2s. c om*/ */ public void batchDone() throws IOException { synchronized (m_state) { if (m_state != STATE_CREATING) { throw new IOException("index is not being re-created"); } } // try to unlock this dir : for unexpected shutdown try { if (IndexWriter.isLocked(m_fsDir)) { IndexWriter.unlock(m_fsDir); } } catch (Exception ee) { // ignore } // Tho reports it can happen that the index cannot be created // on disk (GSDEF00012703). Trap this and release the memory // of the ram directory. IndexWriter diskwriter = null; try { // MUST optimize RAMDirectory before writing it to disk. // m_ramIndexWriter.optimize(); // Write all data out to disk, optimize and clean up. diskwriter = getIndexWriter(true); diskwriter.commit(); Directory[] ds = new Directory[] { m_ramdir }; //Directory[] ds = new Directory[] { m_fsDir } ; diskwriter.addIndexes(ds); //diskwriter.optimize(); //diskwriter.close(); // clean cache if have LuceneCache.cleanLuceneCache(m_directory); } catch (IOException ex) { CATEGORY.error("unexpected error when persisting index " + m_directory, ex); throw ex; } catch (Throwable ex) { CATEGORY.error("unexpected error when persisting index " + m_directory, ex); throw new IOException(ex.getMessage()); } finally { IOUtils.closeWhileHandlingException(diskwriter, m_ramIndexWriter, m_ramdir); m_ramIndexWriter = null; m_ramdir = null; m_state = STATE_OPENED; } }
From source file:com.jaeksoft.searchlib.index.WriterLucene.java
License:Open Source License
private void mergeNoLock(IndexDirectory directory) throws SearchLibException { IndexWriter indexWriter = null; try {/* w w w. j av a 2 s .c o m*/ indexWriter = open(); indexWriter.addIndexes(directory.getDirectory()); close(indexWriter); indexWriter = null; indexLucene.reloadNoLock(); } catch (IOException e) { throw new SearchLibException(e); } finally { close(indexWriter); } }
From source file:com.mathworks.xzheng.indexing.Fragments.java
License:Apache License
public void addIndexes() throws Exception { Directory otherDir = null;// w ww. ja v a 2s . co m Directory ramDir = null; Analyzer analyzer = null; IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, analyzer); // START IndexWriter writer = new IndexWriter(otherDir, config); // UPDATE NOTE: addIndexesNoOptimize was renamed to addIndexes writer.addIndexes(new Directory[] { ramDir }); // END }
From source file:com.nearinfinity.blur.mapreduce.BlurReducer.java
License:Apache License
protected void cleanupFromRebuild(Context context) throws IOException, InterruptedException { _writer.commit();// w w w.j a va 2 s . com _writer.close(); IndexReader reader = IndexReader.open(_directory); TableDescriptor descriptor = _blurTask.getTableDescriptor(); Path directoryPath = _blurTask.getDirectoryPath(context); remove(directoryPath); NoLockFactory lockFactory = NoLockFactory.getNoLockFactory(); Directory destDirectory = getDestDirectory(descriptor, directoryPath); destDirectory.setLockFactory(lockFactory); boolean optimize = _blurTask.getOptimize(); if (optimize) { context.setStatus("Starting Copy-Optimize Phase"); IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer); TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy(); policy.setUseCompoundFile(false); long s = System.currentTimeMillis(); IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory), conf); writer.addIndexes(reader); writer.close(); long e = System.currentTimeMillis(); context.setStatus("Copying phase took [" + (e - s) + " ms]"); LOG.info("Copying phase took [" + (e - s) + " ms]"); } else { context.setStatus("Starting Copy-Optimize Phase"); long s = System.currentTimeMillis(); List<String> files = getFilesOrderedBySize(_directory); long totalBytesToCopy = getTotalBytes(_directory); long totalBytesCopied = 0; long startTime = System.currentTimeMillis(); for (String file : files) { totalBytesCopied += copy(_directory, destDirectory, file, file, context, totalBytesCopied, totalBytesToCopy, startTime); } long e = System.currentTimeMillis(); context.setStatus("Copying phase took [" + (e - s) + " ms]"); LOG.info("Copying phase took [" + (e - s) + " ms]"); } }
From source file:com.plug.Plug_8_5_2.java
License:Apache License
private void reindexTermbase(DbServer dbServer, HashMap<String, String> companys) throws Exception { log.info("Start upgrading Lucene index for termbase"); TermbaseHandler h = new TermbaseHandler(); List<Termbase> tbs = dbServer.getDbUtil().query(TermbaseHandler.SQL, h); m_analyzer = new NgramAnalyzer(3); for (Termbase tb : tbs) { if (tb.getCOMPANYID().equals(LuceneConstants.SUPER_COMPANY_ID)) { continue; }//from w ww . jav a2 s . c o m String cname = companys.get(tb.getCOMPANYID()); File termDir = new File(fileStorageDir, cname + "/TB-" + tb.getTB_NAME()); // check re-indexed if (isIndexedBefore(termDir, tb.getTB_NAME())) { logAlreadyIndex(tb.getTB_NAME()); continue; } showMsg(cname, tb.getTB_NAME(), false); // 1 delete old term base indexes logDeleteFile(termDir.getAbsolutePath()); deleteFile(termDir.getAbsolutePath()); // 2 create new empty dir termDir.mkdirs(); Definition dif = new Definition(tb.getTB_DEFINITION()); List<Index> indexs = dif.getIndexes(); for (Index index : indexs) { // 3 write index into ram RAMDirectory ramdir = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, m_analyzer); config.setOpenMode(OpenMode.CREATE_OR_APPEND); IndexWriter ramIndexWriter = new IndexWriter(ramdir, config); if (index != null && "fuzzy".equalsIgnoreCase(index.getType())) { String folder = index.getLanguageName() + "-" + index.getLocale() + "-TERM"; File indexFolder = new File(termDir, folder); m_directory = indexFolder.getAbsolutePath(); m_fsDir = new SimpleFSDirectory(indexFolder); String sql = TermHandler.generateSQL(tb.getTBID(), index.getLanguageName()); TermHandler termH = new TermHandler(); List<Document> docs = dbServer.getDbUtil().query(sql, termH); for (Document doc : docs) { ramIndexWriter.addDocument(doc); ramIndexWriter.commit(); } // 4 write index from ram into disk IndexWriter diskwriter = getIndexWriter(true); diskwriter.commit(); if (docs != null && docs.size() > 0) { Directory[] ds = new Directory[] { ramdir }; diskwriter.addIndexes(ds); diskwriter.commit(); } // 5 close index writer IOUtils.closeWhileHandlingException(ramIndexWriter); IOUtils.closeWhileHandlingException(diskwriter); ramIndexWriter = null; ramdir = null; } } writeTagFile(termDir, tb.getTB_NAME()); } log.info("End upgrading Lucene index for termbase"); }