List of usage examples for org.apache.lucene.index MultiReader MultiReader
public MultiReader(IndexReader... subReaders) throws IOException
Construct a MultiReader aggregating the named set of (sub)readers.
From source file:org.apache.nutch.tools.PruneIndexTool.java
License:Apache License
/** * Create an instance of the tool, and open all input indexes. * @param indexDirs directories with input indexes. At least one valid index must * exist, otherwise an Exception is thrown. * @param queries pruning queries. Each query will be processed in turn, and the * length of the array must be at least one, otherwise an Exception is thrown. * @param checkers if not null, they will be used to perform additional * checks on matching documents - each checker's method {@link PruneChecker#isPrunable(Query, IndexReader, int)} * will be called in turn, for each matching document, and if it returns true this means that * the document should be deleted. A logical AND is performed on the results returned * by all checkers (which means that if one of them returns false, the document will * not be deleted)./*w w w . j av a 2 s .c om*/ * @param unlock if true, and if any of the input indexes is locked, forcibly * unlock it. Use with care, only when you are sure that other processes don't * modify the index at the same time. * @param dryrun if set to true, don't change the index, just show what would be done. * If false, perform all actions, changing indexes as needed. Note: dryrun doesn't prevent * PruneCheckers from performing changes or causing any other side-effects. * @throws Exception */ public PruneIndexTool(File[] indexDirs, Query[] queries, PruneChecker[] checkers, boolean unlock, boolean dryrun) throws Exception { if (indexDirs == null || queries == null) throw new Exception("Invalid arguments."); if (indexDirs.length == 0 || queries.length == 0) throw new Exception("Nothing to do."); this.queries = queries; this.checkers = checkers; this.dryrun = dryrun; if (dryrun) dr = "[DRY RUN] "; int numIdx = 0; if (indexDirs.length == 1) { Directory dir = FSDirectory.getDirectory(indexDirs[0], false); if (IndexReader.isLocked(dir)) { if (!unlock) { throw new Exception("Index " + indexDirs[0] + " is locked."); } if (!dryrun) { IndexReader.unlock(dir); if (LOG.isDebugEnabled()) { LOG.debug(" - had to unlock index in " + dir); } } } reader = IndexReader.open(dir); numIdx = 1; } else { Directory dir; Vector indexes = new Vector(indexDirs.length); for (int i = 0; i < indexDirs.length; i++) { try { dir = FSDirectory.getDirectory(indexDirs[i], false); if (IndexReader.isLocked(dir)) { if (!unlock) { if (LOG.isWarnEnabled()) { LOG.warn(dr + "Index " + indexDirs[i] + " is locked. Skipping..."); } continue; } if (!dryrun) { IndexReader.unlock(dir); if (LOG.isDebugEnabled()) { LOG.debug(" - had to unlock index in " + dir); } } } IndexReader r = IndexReader.open(dir); indexes.add(r); numIdx++; } catch (Exception e) { if (LOG.isWarnEnabled()) { LOG.warn(dr + "Invalid index in " + indexDirs[i] + " - skipping..."); } } } if (indexes.size() == 0) throw new Exception("No input indexes."); IndexReader[] readers = (IndexReader[]) indexes.toArray(new IndexReader[0]); reader = new MultiReader(readers); } if (LOG.isInfoEnabled()) { LOG.info(dr + "Opened " + numIdx + " index(es) with total " + reader.numDocs() + " documents."); } searcher = new IndexSearcher(reader); }
From source file:org.apache.solr.search.TestDocSet.java
License:Apache License
public IndexReader dummyMultiReader(int nSeg, int maxDoc) throws IOException { if (nSeg == 1 && rand.nextBoolean()) return dummyIndexReader(rand.nextInt(maxDoc)); IndexReader[] subs = new IndexReader[rand.nextInt(nSeg) + 1]; for (int i = 0; i < subs.length; i++) { subs[i] = dummyIndexReader(rand.nextInt(maxDoc)); }/*w w w . j a v a 2 s .co m*/ MultiReader mr = new MultiReader(subs); return mr; }
From source file:org.elasticsearch.common.lucene.IndexCacheableQueryTests.java
License:Apache License
public void testBasics() throws IOException { DummyIndexCacheableQuery query = new DummyIndexCacheableQuery(); QueryUtils.check(query);//w ww . j a va 2s . co m Query rewritten = query.rewrite(new MultiReader(new IndexReader[0])); QueryUtils.check(rewritten); QueryUtils.checkUnequal(query, rewritten); Query rewritten2 = query.rewrite(new MultiReader(new IndexReader[0])); QueryUtils.check(rewritten2); QueryUtils.checkUnequal(rewritten, rewritten2); }
From source file:org.eu.bitzone.Leia.java
License:Apache License
/** * Open indicated index and re-initialize all GUI and plugins. * * @param pName path to index/*from w w w.j ava 2 s . c o m*/ * @param force if true, and the index is locked, unlock it first. If false, and the index is locked, an error will be * reported. * @param readOnly open in read-only mode, and disallow modifications. */ public void openIndex(final String name, final boolean force, final String dirImpl, final boolean ro, final boolean ramdir, final boolean keepCommits, final IndexCommit point, final int tiiDivisor) { pName = name; readOnly = ro; removeAll(); final File baseFileDir = new File(name); this.baseDir = baseFileDir.toString(); addComponent(this, "/xml/luke.xml", null, null); statmsg = find("statmsg"); if (dir != null) { try { if (ir != null) { ir.close(); } if (ar != null) { ar.close(); } } catch (final Exception e) { } ; try { if (dir != null) { dir.close(); } } catch (final Exception e) { } ; } final ArrayList<Directory> dirs = new ArrayList<Directory>(); Throwable lastException = null; try { Directory d = openDirectory(dirImpl, pName, false); if (IndexWriter.isLocked(d)) { if (!ro) { if (force) { IndexWriter.unlock(d); } else { errorMsg("Index is locked. Try 'Force unlock' when opening."); d.close(); d = null; return; } } } boolean existsSingle = false; // IR.indexExists doesn't report the cause of error try { new SegmentInfos().read(d); existsSingle = true; } catch (final Throwable e) { e.printStackTrace(); lastException = e; // } if (!existsSingle) { // try multi final File[] files = baseFileDir.listFiles(); for (final File f : files) { if (f.isFile()) { continue; } Directory d1 = openDirectory(dirImpl, f.toString(), false); if (IndexWriter.isLocked(d1)) { if (!ro) { if (force) { IndexWriter.unlock(d1); } else { errorMsg("Index is locked. Try 'Force unlock' when opening."); d1.close(); d1 = null; return; } } } existsSingle = false; try { new SegmentInfos().read(d1); existsSingle = true; } catch (final Throwable e) { lastException = e; e.printStackTrace(); } if (!existsSingle) { d1.close(); continue; } dirs.add(d1); } } else { dirs.add(d); } if (dirs.size() == 0) { if (lastException != null) { errorMsg( "Invalid directory at the location, check console for more information. Last exception:\n" + lastException.toString()); } else { errorMsg( "No valid directory at the location, try another location.\nCheck console for other possible causes."); } return; } if (ramdir) { showStatus("Loading index into RAMDirectory ..."); final Directory dir1 = new RAMDirectory(); final IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40)); final IndexWriter iw1 = new IndexWriter(dir1, cfg); iw1.addIndexes(dirs.toArray(new Directory[dirs.size()])); iw1.close(); showStatus("RAMDirectory loading done!"); if (dir != null) { dir.close(); } dirs.clear(); dirs.add(dir1); } IndexDeletionPolicy policy; if (keepCommits) { policy = new KeepAllIndexDeletionPolicy(); } else { policy = new KeepLastIndexDeletionPolicy(); } final ArrayList<DirectoryReader> readers = new ArrayList<DirectoryReader>(); for (final Directory dd : dirs) { DirectoryReader reader; if (tiiDivisor > 1) { reader = DirectoryReader.open(dd, tiiDivisor); } else { reader = DirectoryReader.open(dd); } readers.add(reader); } if (readers.size() == 1) { ir = readers.get(0); dir = ((DirectoryReader) ir).directory(); } else { ir = new MultiReader(readers.toArray(new IndexReader[readers.size()])); } is = new IndexSearcher(ir); // XXX slowAccess = false; initOverview(); initPlugins(); showStatus("Index successfully open."); } catch (final Exception e) { e.printStackTrace(); errorMsg(e.getMessage()); return; } }
From source file:org.getopt.luke.Luke.java
License:Apache License
/** * Open indicated index and re-initialize all GUI and plugins. * @param name path to index/* ww w . java2 s. c om*/ * @param force if true, and the index is locked, unlock it first. If false, and * the index is locked, an error will be reported. * @param ro open in read-only mode, and disallow modifications. */ public void openIndex(String name, boolean force, String dirImpl, boolean ro, boolean ramdir, boolean keepCommits, IndexCommit point, int tiiDivisor) { pName = name; readOnly = ro; removeAll(); File baseFileDir = new File(name); this.baseDir = baseFileDir.toString(); addComponent(this, "/xml/luke.xml", null, null); statmsg = find("statmsg"); if (dir != null) { try { if (ir != null) ir.close(); if (ar != null) ar.close(); } catch (Exception e) { } ; try { if (dir != null) dir.close(); } catch (Exception e) { } ; } ArrayList<Directory> dirs = new ArrayList<Directory>(); Throwable lastException = null; try { Directory d = openDirectory(dirImpl, pName, false); if (IndexWriter.isLocked(d)) { if (!ro) { if (force) { IndexWriter.unlock(d); } else { errorMsg("Index is locked. Try 'Force unlock' when opening."); d.close(); d = null; return; } } } boolean existsSingle = false; // IR.indexExists doesn't report the cause of error try { new SegmentInfos().read(d); existsSingle = true; } catch (Throwable e) { e.printStackTrace(); lastException = e; // } if (!existsSingle) { // try multi File[] files = baseFileDir.listFiles(); for (File f : files) { if (f.isFile()) { continue; } Directory d1 = openDirectory(dirImpl, f.toString(), false); if (IndexWriter.isLocked(d1)) { if (!ro) { if (force) { IndexWriter.unlock(d1); } else { errorMsg("Index is locked. Try 'Force unlock' when opening."); d1.close(); d1 = null; return; } } } existsSingle = false; try { new SegmentInfos().read(d1); existsSingle = true; } catch (Throwable e) { lastException = e; e.printStackTrace(); } if (!existsSingle) { d1.close(); continue; } dirs.add(d1); } } else { dirs.add(d); } if (dirs.size() == 0) { if (lastException != null) { errorMsg( "Invalid directory at the location, check console for more information. Last exception:\n" + lastException.toString()); } else { errorMsg( "No valid directory at the location, try another location.\nCheck console for other possible causes."); } return; } if (ramdir) { showStatus("Loading index into RAMDirectory ..."); Directory dir1 = new RAMDirectory(); IndexWriterConfig cfg = new IndexWriterConfig(LV, new WhitespaceAnalyzer(LV)); IndexWriter iw1 = new IndexWriter(dir1, cfg); iw1.addIndexes((Directory[]) dirs.toArray(new Directory[dirs.size()])); iw1.close(); showStatus("RAMDirectory loading done!"); if (dir != null) dir.close(); dirs.clear(); dirs.add(dir1); } IndexDeletionPolicy policy; if (keepCommits) { policy = new KeepAllIndexDeletionPolicy(); } else { policy = new KeepLastIndexDeletionPolicy(); } ArrayList<DirectoryReader> readers = new ArrayList<DirectoryReader>(); for (Directory dd : dirs) { DirectoryReader reader; if (tiiDivisor > 1) { reader = DirectoryReader.open(dd, tiiDivisor); } else { reader = DirectoryReader.open(dd); } readers.add(reader); } if (readers.size() == 1) { ir = readers.get(0); dir = ((DirectoryReader) ir).directory(); } else { ir = new MultiReader((IndexReader[]) readers.toArray(new IndexReader[readers.size()])); } is = new IndexSearcher(ir); // XXX slowAccess = false; initOverview(); initPlugins(); showStatus("Index successfully open."); } catch (Exception e) { e.printStackTrace(); errorMsg(e.getMessage()); return; } }
From source file:org.kie.uberfire.metadata.backend.lucene.index.LuceneIndexManager.java
License:Apache License
public IndexSearcher getIndexSearcher(final ClusterSegment... clusterSegments) { final Set<KCluster> clusters; if (clusterSegments == null || clusterSegments.length == 0) { clusters = new HashSet<KCluster>(indexes.keySet()); } else {//from w ww . j av a2 s .c om clusters = new HashSet<KCluster>(clusterSegments.length); for (final ClusterSegment clusterSegment : clusterSegments) { clusters.add(new KClusterImpl(clusterSegment.getClusterId())); } } final Collection<IndexReader> readers = new ArrayList<IndexReader>(clusters.size()); for (final KCluster cluster : clusters) { final LuceneIndex index = indexes.get(cluster); readers.add(index.nrtReader()); } try { return new SearcherFactory() .newSearcher(new MultiReader(readers.toArray(new IndexReader[readers.size()]))); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.lexevs.dao.index.lucenesupport.MultiBaseLuceneIndexTemplate.java
License:Open Source License
protected MultiReader createReader(IndexReader[] namedDirectories) throws Exception { return new MultiReader(namedDirectories); }
From source file:org.lexevs.dao.index.lucenesupport.MultiBaseLuceneIndexTemplate.java
License:Open Source License
protected MultiReader createIndexReader(List<NamedDirectory> namedDirectories) throws Exception { List<IndexReader> readers = new ArrayList<IndexReader>(); for (NamedDirectory dir : namedDirectories) { readers.add(dir.getIndexReader()); }//from w ww.ja va2 s. c om return new MultiReader(readers.toArray(new IndexReader[readers.size()])); }
From source file:org.maven.ide.eclipse.index.Indexer.java
License:Apache License
public Map search(File[] indexes, String query, String field) throws IOException { if (query == null || query.length() == 0) { return Collections.EMPTY_MAP; }/* w w w .j av a 2s .com*/ Query q; if (query.indexOf('*') > -1) { q = new WildcardQuery(new Term(field, query)); } else { String[] terms = query.split("[\\. -/\\\\]"); int len = terms.length; if (len > 1) { q = new PhraseQuery(); for (int i = 0; i < len; i++) { ((PhraseQuery) q).add(new Term(field, terms[i])); } } else { // q = new TermQuery(new Term(field, query)); q = new WildcardQuery(new Term(field, query + "*")); } } IndexReader[] readers = new IndexReader[indexes.length]; try { for (int i = 0; i < indexes.length; i++) { readers[i] = IndexReader.open(indexes[i]); } IndexSearcher searcher = new IndexSearcher(new MultiReader(readers)); Hits hits = searcher.search(q); if (hits == null || hits.length() == 0) { return Collections.EMPTY_MAP; } return sortResults(query, field, hits); } finally { for (int i = 0; i < readers.length; i++) { try { readers[i].close(); } catch (IOException ex) { // ignore } } } }
From source file:org.netbeans.modules.jackpot30.impl.duplicates.ComputeDuplicates.java
License:Open Source License
public Iterator<? extends DuplicateDescription> computeDuplicates(Set<URL> forURLs, ProgressHandle progress, AtomicBoolean cancel) throws IOException { Map<IndexReader, FileObject> readers2Roots = new LinkedHashMap<IndexReader, FileObject>(); progress.progress("Updating indices"); for (URL u : forURLs) { try {//w w w . j a va 2s.c o m //TODO: needs to be removed for server mode new DuplicatesCustomIndexerImpl.FactoryImpl().updateIndex(u, cancel); //TODO: show updating progress to the user File cacheRoot = cacheRoot(u); File dir = new File(cacheRoot, DuplicatesIndex.NAME); if (dir.listFiles() != null && dir.listFiles().length > 0) { IndexReader reader = IndexReader.open(FSDirectory.open(dir), true); readers2Roots.put(reader, URLMapper.findFileObject(u)); } } catch (IOException ex) { Exceptions.printStackTrace(ex); } } progress.progress("Searching for duplicates"); MultiReader r = new MultiReader(readers2Roots.keySet().toArray(new IndexReader[0])); List<String> dd = new ArrayList<String>(getDuplicatedValues(r, "duplicatesGeneralized", cancel)); sortHashes(dd); //TODO: only show valuable duplicates?: // dd = dd.subList(0, dd.size() / 10 + 1); return new DuplicatesIterator(readers2Roots, dd, 2); }