List of usage examples for org.apache.lucene.index IndexReader maxDoc
public abstract int maxDoc();
From source file:org.elasticsearch.index.search.child.ChildrenConstantScoreQueryTests.java
License:Apache License
@Test public void testRandom() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); int numUniqueChildValues = 1 + random().nextInt(TEST_NIGHTLY ? 10000 : 1000); String[] childValues = new String[numUniqueChildValues]; for (int i = 0; i < numUniqueChildValues; i++) { childValues[i] = Integer.toString(i); }//from w w w. j a va 2s. com IntOpenHashSet filteredOrDeletedDocs = new IntOpenHashSet(); int childDocId = 0; int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); ObjectObjectOpenHashMap<String, NavigableSet<String>> childValueToParentIds = new ObjectObjectOpenHashMap<String, NavigableSet<String>>(); for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { boolean markParentAsDeleted = rarely(); boolean filterMe = rarely(); String parent = Integer.toString(parentDocId); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); if (markParentAsDeleted) { filteredOrDeletedDocs.add(parentDocId); document.add(new StringField("delete", "me", Field.Store.NO)); } if (filterMe) { filteredOrDeletedDocs.add(parentDocId); document.add(new StringField("filter", "me", Field.Store.NO)); } indexWriter.addDocument(document); int numChildDocs; if (rarely()) { numChildDocs = random().nextInt(TEST_NIGHTLY ? 100 : 25); } else { numChildDocs = random().nextInt(TEST_NIGHTLY ? 40 : 10); } for (int i = 0; i < numChildDocs; i++) { boolean markChildAsDeleted = rarely(); String childValue = childValues[random().nextInt(childValues.length)]; document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("child", Integer.toString(childDocId)), Field.Store.NO)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); document.add( new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); document.add(new StringField("field1", childValue, Field.Store.NO)); if (markChildAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } indexWriter.addDocument(document); if (!markChildAsDeleted) { NavigableSet<String> parentIds; if (childValueToParentIds.containsKey(childValue)) { parentIds = childValueToParentIds.lget(); } else { childValueToParentIds.put(childValue, parentIds = new TreeSet<String>()); } if (!markParentAsDeleted && !filterMe) { parentIds.add(parent); } } } } // Delete docs that are marked to be deleted. indexWriter.deleteDocuments(new Term("delete", "me")); indexWriter.commit(); IndexReader indexReader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(indexReader); Engine.Searcher engineSearcher = new Engine.SimpleSearcher( ChildrenConstantScoreQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); ParentFieldMapper parentFieldMapper = SearchContext.current().mapperService().documentMapper("child") .parentFieldMapper(); ParentChildIndexFieldData parentChildIndexFieldData = SearchContext.current().fieldData() .getForField(parentFieldMapper); Filter rawParentFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "parent")); Filter rawFilterMe = new NotFilter(new TermFilter(new Term("filter", "me"))); int max = numUniqueChildValues / 4; for (int i = 0; i < max; i++) { // Randomly pick a cached version: there is specific logic inside ChildrenQuery that deals with the fact // that deletes are applied at the top level when filters are cached. Filter parentFilter; if (random().nextBoolean()) { parentFilter = SearchContext.current().filterCache().cache(rawParentFilter); } else { parentFilter = rawParentFilter; } // Using this in FQ, will invoke / test the Scorer#advance(..) and also let the Weight#scorer not get live docs as acceptedDocs Filter filterMe; if (random().nextBoolean()) { filterMe = SearchContext.current().filterCache().cache(rawFilterMe); } else { filterMe = rawFilterMe; } // Simulate a parent update if (random().nextBoolean()) { int numberOfUpdates = 1 + random().nextInt(TEST_NIGHTLY ? 25 : 5); for (int j = 0; j < numberOfUpdates; j++) { int parentId; do { parentId = random().nextInt(numParentDocs); } while (filteredOrDeletedDocs.contains(parentId)); String parentUid = Uid.createUid("parent", Integer.toString(parentId)); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, parentUid)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, parentUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); indexWriter.addDocument(document); } indexReader.close(); indexReader = DirectoryReader.open(indexWriter.w, true); searcher = new IndexSearcher(indexReader); engineSearcher = new Engine.SimpleSearcher(ChildrenConstantScoreQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); } String childValue = childValues[random().nextInt(numUniqueChildValues)]; TermQuery childQuery = new TermQuery(new Term("field1", childValue)); int shortCircuitParentDocSet = random().nextInt(numParentDocs); Filter nonNestedDocsFilter = random().nextBoolean() ? NonNestedDocsFilter.INSTANCE : null; Query query; if (random().nextBoolean()) { // Usage in HasChildQueryParser query = new ChildrenConstantScoreQuery(parentChildIndexFieldData, childQuery, "parent", "child", parentFilter, shortCircuitParentDocSet, nonNestedDocsFilter); } else { // Usage in HasChildFilterParser query = new XConstantScoreQuery(new CustomQueryWrappingFilter( new ChildrenConstantScoreQuery(parentChildIndexFieldData, childQuery, "parent", "child", parentFilter, shortCircuitParentDocSet, nonNestedDocsFilter))); } query = new XFilteredQuery(query, filterMe); BitSetCollector collector = new BitSetCollector(indexReader.maxDoc()); searcher.search(query, collector); FixedBitSet actualResult = collector.getResult(); FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc()); if (childValueToParentIds.containsKey(childValue)) { AtomicReader slowAtomicReader = SlowCompositeReaderWrapper.wrap(indexReader); Terms terms = slowAtomicReader.terms(UidFieldMapper.NAME); if (terms != null) { NavigableSet<String> parentIds = childValueToParentIds.lget(); TermsEnum termsEnum = terms.iterator(null); DocsEnum docsEnum = null; for (String id : parentIds) { TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(Uid.createUidAsBytes("parent", id)); if (seekStatus == TermsEnum.SeekStatus.FOUND) { docsEnum = termsEnum.docs(slowAtomicReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); expectedResult.set(docsEnum.nextDoc()); } else if (seekStatus == TermsEnum.SeekStatus.END) { break; } } } } assertBitSet(actualResult, expectedResult, searcher); } indexWriter.close(); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.search.child.ChildrenQueryTests.java
License:Apache License
@Test public void testRandom() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); int numUniqueChildValues = 1 + random().nextInt(TEST_NIGHTLY ? 6000 : 600); String[] childValues = new String[numUniqueChildValues]; for (int i = 0; i < numUniqueChildValues; i++) { childValues[i] = Integer.toString(i); }//from w w w . java 2 s. co m IntOpenHashSet filteredOrDeletedDocs = new IntOpenHashSet(); int childDocId = 0; int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); ObjectObjectOpenHashMap<String, NavigableMap<String, FloatArrayList>> childValueToParentIds = new ObjectObjectOpenHashMap<String, NavigableMap<String, FloatArrayList>>(); for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { boolean markParentAsDeleted = rarely(); boolean filterMe = rarely(); String parent = Integer.toString(parentDocId); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); if (markParentAsDeleted) { filteredOrDeletedDocs.add(parentDocId); document.add(new StringField("delete", "me", Field.Store.NO)); } if (filterMe) { filteredOrDeletedDocs.add(parentDocId); document.add(new StringField("filter", "me", Field.Store.NO)); } indexWriter.addDocument(document); int numChildDocs; if (rarely()) { numChildDocs = random().nextInt(TEST_NIGHTLY ? 100 : 25); } else { numChildDocs = random().nextInt(TEST_NIGHTLY ? 40 : 10); } for (int i = 0; i < numChildDocs; i++) { boolean markChildAsDeleted = rarely(); String childValue = childValues[random().nextInt(childValues.length)]; document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("child", Integer.toString(childDocId)), Field.Store.NO)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); document.add( new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); document.add(new StringField("field1", childValue, Field.Store.NO)); if (markChildAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } indexWriter.addDocument(document); if (!markChildAsDeleted) { NavigableMap<String, FloatArrayList> parentIdToChildScores; if (childValueToParentIds.containsKey(childValue)) { parentIdToChildScores = childValueToParentIds.lget(); } else { childValueToParentIds.put(childValue, parentIdToChildScores = new TreeMap<String, FloatArrayList>()); } if (!markParentAsDeleted && !filterMe) { FloatArrayList childScores = parentIdToChildScores.get(parent); if (childScores == null) { parentIdToChildScores.put(parent, childScores = new FloatArrayList()); } childScores.add(1f); } } } } // Delete docs that are marked to be deleted. indexWriter.deleteDocuments(new Term("delete", "me")); indexWriter.commit(); IndexReader indexReader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(indexReader); Engine.Searcher engineSearcher = new Engine.SimpleSearcher(ChildrenQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); ParentFieldMapper parentFieldMapper = SearchContext.current().mapperService().documentMapper("child") .parentFieldMapper(); ParentChildIndexFieldData parentChildIndexFieldData = SearchContext.current().fieldData() .getForField(parentFieldMapper); Filter rawParentFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "parent")); Filter rawFilterMe = new NotFilter(new TermFilter(new Term("filter", "me"))); int max = numUniqueChildValues / 4; for (int i = 0; i < max; i++) { // Randomly pick a cached version: there is specific logic inside ChildrenQuery that deals with the fact // that deletes are applied at the top level when filters are cached. Filter parentFilter; if (random().nextBoolean()) { parentFilter = SearchContext.current().filterCache().cache(rawParentFilter); } else { parentFilter = rawParentFilter; } // Using this in FQ, will invoke / test the Scorer#advance(..) and also let the Weight#scorer not get live docs as acceptedDocs Filter filterMe; if (random().nextBoolean()) { filterMe = SearchContext.current().filterCache().cache(rawFilterMe); } else { filterMe = rawFilterMe; } // Simulate a parent update if (random().nextBoolean()) { int numberOfUpdates = 1 + random().nextInt(TEST_NIGHTLY ? 25 : 5); for (int j = 0; j < numberOfUpdates; j++) { int parentId; do { parentId = random().nextInt(numParentDocs); } while (filteredOrDeletedDocs.contains(parentId)); String parentUid = Uid.createUid("parent", Integer.toString(parentId)); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, parentUid)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, parentUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); indexWriter.addDocument(document); } indexReader.close(); indexReader = DirectoryReader.open(indexWriter.w, true); searcher = new IndexSearcher(indexReader); engineSearcher = new Engine.SimpleSearcher(ChildrenConstantScoreQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); } String childValue = childValues[random().nextInt(numUniqueChildValues)]; Query childQuery = new ConstantScoreQuery(new TermQuery(new Term("field1", childValue))); int shortCircuitParentDocSet = random().nextInt(numParentDocs); ScoreType scoreType = ScoreType.values()[random().nextInt(ScoreType.values().length)]; Filter nonNestedDocsFilter = random().nextBoolean() ? NonNestedDocsFilter.INSTANCE : null; Query query = new ChildrenQuery(parentChildIndexFieldData, "parent", "child", parentFilter, childQuery, scoreType, shortCircuitParentDocSet, nonNestedDocsFilter); query = new XFilteredQuery(query, filterMe); BitSetCollector collector = new BitSetCollector(indexReader.maxDoc()); int numHits = 1 + random().nextInt(25); TopScoreDocCollector actualTopDocsCollector = TopScoreDocCollector.create(numHits, false); searcher.search(query, MultiCollector.wrap(collector, actualTopDocsCollector)); FixedBitSet actualResult = collector.getResult(); FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc()); MockScorer mockScorer = new MockScorer(scoreType); TopScoreDocCollector expectedTopDocsCollector = TopScoreDocCollector.create(numHits, false); expectedTopDocsCollector.setScorer(mockScorer); if (childValueToParentIds.containsKey(childValue)) { AtomicReader slowAtomicReader = SlowCompositeReaderWrapper.wrap(indexReader); Terms terms = slowAtomicReader.terms(UidFieldMapper.NAME); if (terms != null) { NavigableMap<String, FloatArrayList> parentIdToChildScores = childValueToParentIds.lget(); TermsEnum termsEnum = terms.iterator(null); DocsEnum docsEnum = null; for (Map.Entry<String, FloatArrayList> entry : parentIdToChildScores.entrySet()) { TermsEnum.SeekStatus seekStatus = termsEnum .seekCeil(Uid.createUidAsBytes("parent", entry.getKey())); if (seekStatus == TermsEnum.SeekStatus.FOUND) { docsEnum = termsEnum.docs(slowAtomicReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); expectedResult.set(docsEnum.nextDoc()); mockScorer.scores = entry.getValue(); expectedTopDocsCollector.collect(docsEnum.docID()); } else if (seekStatus == TermsEnum.SeekStatus.END) { break; } } } } assertBitSet(actualResult, expectedResult, searcher); assertTopDocs(actualTopDocsCollector.topDocs(), expectedTopDocsCollector.topDocs()); } indexWriter.close(); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.search.child.ParentConstantScoreQueryTests.java
License:Apache License
@Test public void testRandom() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); int numUniqueParentValues = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); String[] parentValues = new String[numUniqueParentValues]; for (int i = 0; i < numUniqueParentValues; i++) { parentValues[i] = Integer.toString(i); }/*from w w w. ja v a2 s . co m*/ int childDocId = 0; int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 10000 : 1000); ObjectObjectOpenHashMap<String, NavigableSet<String>> parentValueToChildDocIds = new ObjectObjectOpenHashMap<String, NavigableSet<String>>(); IntIntOpenHashMap childIdToParentId = new IntIntOpenHashMap(); for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { boolean markParentAsDeleted = rarely(); String parentValue = parentValues[random().nextInt(parentValues.length)]; String parent = Integer.toString(parentDocId); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", parentValue, Field.Store.NO)); if (markParentAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } indexWriter.addDocument(document); int numChildDocs; if (rarely()) { numChildDocs = random().nextInt(TEST_NIGHTLY ? 100 : 25); } else { numChildDocs = random().nextInt(TEST_NIGHTLY ? 40 : 10); } if (parentDocId == numParentDocs - 1 && childIdToParentId.isEmpty()) { // ensure there is at least one child in the index numChildDocs = Math.max(1, numChildDocs); } for (int i = 0; i < numChildDocs; i++) { boolean markChildAsDeleted = rarely(); boolean filterMe = rarely(); String child = Integer.toString(childDocId++); document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("child", child), Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); document.add( new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); if (markChildAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } if (filterMe) { document.add(new StringField("filter", "me", Field.Store.NO)); } indexWriter.addDocument(document); if (!markParentAsDeleted) { NavigableSet<String> childIds; if (parentValueToChildDocIds.containsKey(parentValue)) { childIds = parentValueToChildDocIds.lget(); } else { parentValueToChildDocIds.put(parentValue, childIds = new TreeSet<String>()); } if (!markChildAsDeleted && !filterMe) { childIdToParentId.put(Integer.valueOf(child), parentDocId); childIds.add(child); } } } } // Delete docs that are marked to be deleted. indexWriter.deleteDocuments(new Term("delete", "me")); indexWriter.commit(); IndexReader indexReader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(indexReader); Engine.Searcher engineSearcher = new Engine.SimpleSearcher(ParentConstantScoreQuery.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); ParentFieldMapper parentFieldMapper = SearchContext.current().mapperService().documentMapper("child") .parentFieldMapper(); ParentChildIndexFieldData parentChildIndexFieldData = SearchContext.current().fieldData() .getForField(parentFieldMapper); TermFilter rawChildrenFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "child")); Filter rawFilterMe = new NotFilter(new TermFilter(new Term("filter", "me"))); int max = numUniqueParentValues / 4; for (int i = 0; i < max; i++) { // Randomly pick a cached version: there is specific logic inside ChildrenQuery that deals with the fact // that deletes are applied at the top level when filters are cached. Filter childrenFilter; if (random().nextBoolean()) { childrenFilter = SearchContext.current().filterCache().cache(rawChildrenFilter); } else { childrenFilter = rawChildrenFilter; } // Using this in FQ, will invoke / test the Scorer#advance(..) and also let the Weight#scorer not get live docs as acceptedDocs Filter filterMe; if (random().nextBoolean()) { filterMe = SearchContext.current().filterCache().cache(rawFilterMe); } else { filterMe = rawFilterMe; } // Simulate a child update if (random().nextBoolean()) { int numberOfUpdates = 1 + random().nextInt(TEST_NIGHTLY ? 25 : 5); int[] childIds = childIdToParentId.keys().toArray(); for (int j = 0; j < numberOfUpdates; j++) { int childId = childIds[random().nextInt(childIds.length)]; String childUid = Uid.createUid("child", Integer.toString(childId)); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, childUid)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, childUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); String parentUid = Uid.createUid("parent", Integer.toString(childIdToParentId.get(childId))); document.add(new StringField(ParentFieldMapper.NAME, parentUid, Field.Store.NO)); indexWriter.addDocument(document); } indexReader.close(); indexReader = DirectoryReader.open(indexWriter.w, true); searcher = new IndexSearcher(indexReader); engineSearcher = new Engine.SimpleSearcher(ParentConstantScoreQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); } String parentValue = parentValues[random().nextInt(numUniqueParentValues)]; TermQuery parentQuery = new TermQuery(new Term("field1", parentValue)); Query query; if (random().nextBoolean()) { // Usage in HasParentQueryParser query = new ParentConstantScoreQuery(parentChildIndexFieldData, parentQuery, "parent", childrenFilter); } else { // Usage in HasParentFilterParser query = new XConstantScoreQuery(new CustomQueryWrappingFilter(new ParentConstantScoreQuery( parentChildIndexFieldData, parentQuery, "parent", childrenFilter))); } query = new XFilteredQuery(query, filterMe); BitSetCollector collector = new BitSetCollector(indexReader.maxDoc()); searcher.search(query, collector); FixedBitSet actualResult = collector.getResult(); FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc()); if (parentValueToChildDocIds.containsKey(parentValue)) { AtomicReader slowAtomicReader = SlowCompositeReaderWrapper.wrap(indexReader); Terms terms = slowAtomicReader.terms(UidFieldMapper.NAME); if (terms != null) { NavigableSet<String> childIds = parentValueToChildDocIds.lget(); TermsEnum termsEnum = terms.iterator(null); DocsEnum docsEnum = null; for (String id : childIds) { TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(Uid.createUidAsBytes("child", id)); if (seekStatus == TermsEnum.SeekStatus.FOUND) { docsEnum = termsEnum.docs(slowAtomicReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); expectedResult.set(docsEnum.nextDoc()); } else if (seekStatus == TermsEnum.SeekStatus.END) { break; } } } } assertBitSet(actualResult, expectedResult, searcher); } indexWriter.close(); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.search.child.ParentQueryTests.java
License:Apache License
@Test public void testRandom() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); int numUniqueParentValues = 1 + random().nextInt(TEST_NIGHTLY ? 6000 : 600); String[] parentValues = new String[numUniqueParentValues]; for (int i = 0; i < numUniqueParentValues; i++) { parentValues[i] = Integer.toString(i); }/*from w w w . j av a2s . com*/ int childDocId = 0; int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); ObjectObjectOpenHashMap<String, NavigableMap<String, Float>> parentValueToChildIds = new ObjectObjectOpenHashMap<String, NavigableMap<String, Float>>(); IntIntOpenHashMap childIdToParentId = new IntIntOpenHashMap(); for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { boolean markParentAsDeleted = rarely(); String parentValue = parentValues[random().nextInt(parentValues.length)]; String parent = Integer.toString(parentDocId); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); document.add(new StringField("field1", parentValue, Field.Store.NO)); if (markParentAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } indexWriter.addDocument(document); int numChildDocs; if (rarely()) { numChildDocs = random().nextInt(TEST_NIGHTLY ? 100 : 25); } else { numChildDocs = random().nextInt(TEST_NIGHTLY ? 40 : 10); } if (parentDocId == numParentDocs - 1 && childIdToParentId.isEmpty()) { // ensure there is at least one child in the index numChildDocs = Math.max(1, numChildDocs); } for (int i = 0; i < numChildDocs; i++) { String child = Integer.toString(childDocId++); boolean markChildAsDeleted = rarely(); boolean filterMe = rarely(); document = new Document(); document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("child", child), Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); document.add( new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); if (markChildAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } if (filterMe) { document.add(new StringField("filter", "me", Field.Store.NO)); } indexWriter.addDocument(document); if (!markParentAsDeleted) { NavigableMap<String, Float> childIdToScore; if (parentValueToChildIds.containsKey(parentValue)) { childIdToScore = parentValueToChildIds.lget(); } else { parentValueToChildIds.put(parentValue, childIdToScore = new TreeMap<String, Float>()); } if (!markChildAsDeleted && !filterMe) { assertFalse("child [" + child + "] already has a score", childIdToScore.containsKey(child)); childIdToScore.put(child, 1f); childIdToParentId.put(Integer.valueOf(child), parentDocId); } } } } // Delete docs that are marked to be deleted. indexWriter.deleteDocuments(new Term("delete", "me")); indexWriter.commit(); IndexReader indexReader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(indexReader); Engine.Searcher engineSearcher = new Engine.SimpleSearcher(ParentQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); ParentFieldMapper parentFieldMapper = SearchContext.current().mapperService().documentMapper("child") .parentFieldMapper(); ParentChildIndexFieldData parentChildIndexFieldData = SearchContext.current().fieldData() .getForField(parentFieldMapper); TermFilter rawChildrenFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "child")); Filter rawFilterMe = new NotFilter(new TermFilter(new Term("filter", "me"))); int max = numUniqueParentValues / 4; for (int i = 0; i < max; i++) { // Randomly pick a cached version: there is specific logic inside ChildrenQuery that deals with the fact // that deletes are applied at the top level when filters are cached. Filter childrenFilter; if (random().nextBoolean()) { childrenFilter = SearchContext.current().filterCache().cache(rawChildrenFilter); } else { childrenFilter = rawChildrenFilter; } // Using this in FQ, will invoke / test the Scorer#advance(..) and also let the Weight#scorer not get live docs as acceptedDocs Filter filterMe; if (random().nextBoolean()) { filterMe = SearchContext.current().filterCache().cache(rawFilterMe); } else { filterMe = rawFilterMe; } // Simulate a child update if (random().nextBoolean()) { int numberOfUpdates = 1 + random().nextInt(TEST_NIGHTLY ? 25 : 5); int[] childIds = childIdToParentId.keys().toArray(); for (int j = 0; j < numberOfUpdates; j++) { int childId = childIds[random().nextInt(childIds.length)]; String childUid = Uid.createUid("child", Integer.toString(childId)); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, childUid)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, childUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); String parentUid = Uid.createUid("parent", Integer.toString(childIdToParentId.get(childId))); document.add(new StringField(ParentFieldMapper.NAME, parentUid, Field.Store.NO)); indexWriter.addDocument(document); } indexReader.close(); indexReader = DirectoryReader.open(indexWriter.w, true); searcher = new IndexSearcher(indexReader); engineSearcher = new Engine.SimpleSearcher(ParentConstantScoreQueryTests.class.getSimpleName(), searcher); ((TestSearchContext) SearchContext.current()) .setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); } String parentValue = parentValues[random().nextInt(numUniqueParentValues)]; Query parentQuery = new ConstantScoreQuery(new TermQuery(new Term("field1", parentValue))); Query query = new ParentQuery(parentChildIndexFieldData, parentQuery, "parent", childrenFilter); query = new XFilteredQuery(query, filterMe); BitSetCollector collector = new BitSetCollector(indexReader.maxDoc()); int numHits = 1 + random().nextInt(25); TopScoreDocCollector actualTopDocsCollector = TopScoreDocCollector.create(numHits, false); searcher.search(query, MultiCollector.wrap(collector, actualTopDocsCollector)); FixedBitSet actualResult = collector.getResult(); FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc()); MockScorer mockScorer = new MockScorer(ScoreType.MAX); // just save one score per parent... mockScorer.scores = new FloatArrayList(); TopScoreDocCollector expectedTopDocsCollector = TopScoreDocCollector.create(numHits, false); expectedTopDocsCollector.setScorer(mockScorer); if (parentValueToChildIds.containsKey(parentValue)) { AtomicReader slowAtomicReader = SlowCompositeReaderWrapper.wrap(indexReader); Terms terms = slowAtomicReader.terms(UidFieldMapper.NAME); if (terms != null) { NavigableMap<String, Float> childIdsAndScore = parentValueToChildIds.lget(); TermsEnum termsEnum = terms.iterator(null); DocsEnum docsEnum = null; for (Map.Entry<String, Float> entry : childIdsAndScore.entrySet()) { TermsEnum.SeekStatus seekStatus = termsEnum .seekCeil(Uid.createUidAsBytes("child", entry.getKey())); if (seekStatus == TermsEnum.SeekStatus.FOUND) { docsEnum = termsEnum.docs(slowAtomicReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); expectedResult.set(docsEnum.nextDoc()); mockScorer.scores.add(entry.getValue()); expectedTopDocsCollector.collect(docsEnum.docID()); mockScorer.scores.clear(); } else if (seekStatus == TermsEnum.SeekStatus.END) { break; } } } } assertBitSet(actualResult, expectedResult, searcher); assertTopDocs(actualTopDocsCollector.topDocs(), expectedTopDocsCollector.topDocs()); } indexWriter.close(); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.search.geo.GeoBoundingBoxFilter.java
License:Apache License
@Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, reader, fieldName);// w w w.j a v a2 s. c o m //checks to see if bounding box crosses 180 degrees if (topLeft.lon > bottomRight.lon) { return new GetDocSet(reader.maxDoc()) { @Override public boolean isCacheable() { // not cacheable for several reasons: // 1. It is only relevant when _cache is set to true, and then, we really want to create in mem bitset // 2. Its already fast without in mem bitset, since it works with field data return false; } @Override public boolean get(int doc) throws IOException { if (!fieldData.hasValue(doc)) { return false; } if (fieldData.multiValued()) { double[] lats = fieldData.latValues(doc); double[] lons = fieldData.lonValues(doc); for (int i = 0; i < lats.length; i++) { double lat = lats[i]; double lon = lons[i]; if (((topLeft.lon <= lon && 180 >= lon) || (-180 <= lon && bottomRight.lon >= lon)) && (topLeft.lat >= lat && bottomRight.lat <= lat)) { return true; } } } else { double lat = fieldData.latValue(doc); double lon = fieldData.lonValue(doc); if (((topLeft.lon <= lon && 180 >= lon) || (-180 <= lon && bottomRight.lon >= lon)) && (topLeft.lat >= lat && bottomRight.lat <= lat)) { return true; } } return false; } }; } else { return new GetDocSet(reader.maxDoc()) { @Override public boolean isCacheable() { // not cacheable for several reasons: // 1. It is only relevant when _cache is set to true, and then, we really want to create in mem bitset // 2. Its already fast without in mem bitset, since it works with field data return false; } @Override public boolean get(int doc) throws IOException { if (!fieldData.hasValue(doc)) { return false; } if (fieldData.multiValued()) { double[] lats = fieldData.latValues(doc); double[] lons = fieldData.lonValues(doc); for (int i = 0; i < lats.length; i++) { if (topLeft.lon <= lons[i] && bottomRight.lon >= lons[i] && topLeft.lat >= lats[i] && bottomRight.lat <= lats[i]) { return true; } } } else { double lat = fieldData.latValue(doc); double lon = fieldData.lonValue(doc); if (topLeft.lon <= lon && bottomRight.lon >= lon && topLeft.lat >= lat && bottomRight.lat <= lat) { return true; } } return false; } }; } }
From source file:org.elasticsearch.index.search.UidFilter.java
License:Apache License
@Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { BloomFilter filter = bloomCache.filter(reader, UidFieldMapper.NAME, true); OpenBitSet set = null;//from w ww.j a v a 2 s . c o m TermDocs td = null; try { for (Term uid : uids) { UnicodeUtil.UTF8Result utf8 = Unicode.fromStringAsUtf8(uid.text()); if (!filter.isPresent(utf8.result, 0, utf8.length)) { continue; } if (td == null) { td = reader.termDocs(); } td.seek(uid); while (td.next()) { if (set == null) { set = new OpenBitSet(reader.maxDoc()); } set.fastSet(td.doc()); } } } finally { if (td != null) { td.close(); } } return set; }
From source file:org.elasticsearch.indices.recovery.RecoverySourceHandlerTests.java
License:Apache License
public void testSendFiles() throws Throwable { Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1) .put("indices.recovery.concurrent_small_file_streams", 1).build(); final RecoverySettings recoverySettings = new RecoverySettings(settings, service); StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),// w ww.ja va 2s . co m new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomLong()); Store store = newStore(createTempDir()); RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> { }, recoverySettings.getChunkSize().bytesAsInt(), logger); Directory dir = store.directory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig()); int numDocs = randomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { Document document = new Document(); document.add(new StringField("id", Integer.toString(i), Field.Store.YES)); document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED)); writer.addDocument(document); } writer.commit(); writer.close(); Store.MetadataSnapshot metadata = store.getMetadata(null); List<StoreFileMetaData> metas = new ArrayList<>(); for (StoreFileMetaData md : metadata) { metas.add(md); } Store targetStore = newStore(createTempDir()); handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> { try { return new IndexOutputOutputStream( targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) { @Override public void close() throws IOException { super.close(); targetStore.directory().sync(Collections.singleton(md.name())); // sync otherwise MDW will mess with it } }; } catch (IOException e) { throw new RuntimeException(e); } }); Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata(null); Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata); assertEquals(metas.size(), recoveryDiff.identical.size()); assertEquals(0, recoveryDiff.different.size()); assertEquals(0, recoveryDiff.missing.size()); IndexReader reader = DirectoryReader.open(targetStore.directory()); assertEquals(numDocs, reader.maxDoc()); IOUtils.close(reader, store, targetStore); }
From source file:org.elasticsearch.search.aggregations.bucket.composite.SingleDimensionValuesSource.java
License:Apache License
/** * Returns true if a {@link SortedDocsProducer} should be used to optimize the execution. *//*from w ww .jav a 2s .c om*/ protected boolean checkIfSortedDocsIsApplicable(IndexReader reader, MappedFieldType fieldType) { if (fieldType == null || fieldType.indexOptions() == IndexOptions.NONE || // inverse of the natural order reverseMul == -1) { return false; } if (reader.hasDeletions() && (reader.numDocs() == 0 || (double) reader.numDocs() / (double) reader.maxDoc() < 0.5)) { // do not use the index if it has more than 50% of deleted docs return false; } return true; }
From source file:org.elasticsearch.search.aggregations.bucket.composite.SingleDimensionValuesSourceTests.java
License:Apache License
private static IndexReader mockIndexReader(int maxDoc, int numDocs) { IndexReader reader = mock(IndexReader.class); when(reader.hasDeletions()).thenReturn(maxDoc - numDocs > 0); when(reader.maxDoc()).thenReturn(maxDoc); when(reader.numDocs()).thenReturn(numDocs); return reader; }
From source file:org.elasticsearch.search.suggest.phrase.DirectCandidateGenerator.java
License:Apache License
public DirectCandidateGenerator(DirectSpellChecker spellchecker, String field, SuggestMode suggestMode, IndexReader reader, double nonErrorLikelihood, int numCandidates, Analyzer preFilter, Analyzer postFilter, Terms terms) throws IOException { if (terms == null) { throw new ElasticsearchIllegalArgumentException("generator field [" + field + "] doesn't exist"); }//from ww w . j a v a 2 s .c o m this.spellchecker = spellchecker; this.field = field; this.numCandidates = numCandidates; this.suggestMode = suggestMode; this.reader = reader; final long dictSize = terms.getSumTotalTermFreq(); this.useTotalTermFrequency = dictSize != -1; this.dictSize = dictSize == -1 ? reader.maxDoc() : dictSize; this.preFilter = preFilter; this.postFilter = postFilter; this.nonErrorLikelihood = nonErrorLikelihood; float thresholdFrequency = spellchecker.getThresholdFrequency(); this.frequencyPlateau = thresholdFrequency >= 1.0f ? (int) thresholdFrequency : (int) (dictSize * thresholdFrequency); termsEnum = terms.iterator(null); }