List of usage examples for org.apache.lucene.index NumericDocValues longValue
public abstract long longValue() throws IOException;
From source file:org.apache.solr.search.SolrDocumentFetcher.java
License:Apache License
/** * This will fetch and add the docValues fields to a given SolrDocument/SolrInputDocument * * @param doc/*from w w w . j a v a2 s .co m*/ * A SolrDocument or SolrInputDocument instance where docValues will be added * @param docid * The lucene docid of the document to be populated * @param fields * The list of docValues fields to be decorated */ public void decorateDocValueFields(@SuppressWarnings("rawtypes") SolrDocumentBase doc, int docid, Set<String> fields) throws IOException { final List<LeafReaderContext> leafContexts = searcher.getLeafContexts(); final int subIndex = ReaderUtil.subIndex(docid, leafContexts); final int localId = docid - leafContexts.get(subIndex).docBase; final LeafReader leafReader = leafContexts.get(subIndex).reader(); for (String fieldName : fields) { final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldName); if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) { log.warn("Couldn't decorate docValues for field: [{}], schemaField: [{}]", fieldName, schemaField); continue; } FieldInfo fi = searcher.getFieldInfos().fieldInfo(fieldName); if (fi == null) { continue; // Searcher doesn't have info about this field, hence ignore it. } final DocValuesType dvType = fi.getDocValuesType(); switch (dvType) { case NUMERIC: final NumericDocValues ndv = leafReader.getNumericDocValues(fieldName); if (ndv == null) { continue; } Long val; if (ndv.advanceExact(localId)) { val = ndv.longValue(); } else { continue; } Object newVal = val; if (schemaField.getType().isPointField()) { // TODO: Maybe merge PointField with TrieFields here NumberType type = schemaField.getType().getNumberType(); switch (type) { case INTEGER: newVal = val.intValue(); break; case LONG: newVal = val.longValue(); break; case FLOAT: newVal = Float.intBitsToFloat(val.intValue()); break; case DOUBLE: newVal = Double.longBitsToDouble(val); break; case DATE: newVal = new Date(val); break; default: throw new AssertionError("Unexpected PointType: " + type); } } else { if (schemaField.getType() instanceof TrieIntField) { newVal = val.intValue(); } else if (schemaField.getType() instanceof TrieFloatField) { newVal = Float.intBitsToFloat(val.intValue()); } else if (schemaField.getType() instanceof TrieDoubleField) { newVal = Double.longBitsToDouble(val); } else if (schemaField.getType() instanceof TrieDateField) { newVal = new Date(val); } else if (schemaField.getType() instanceof EnumField) { newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue()); } } doc.addField(fieldName, newVal); break; case BINARY: BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName); if (bdv == null) { continue; } BytesRef value; if (bdv.advanceExact(localId)) { value = BytesRef.deepCopyOf(bdv.binaryValue()); } else { continue; } doc.addField(fieldName, value); break; case SORTED: SortedDocValues sdv = leafReader.getSortedDocValues(fieldName); if (sdv == null) { continue; } if (sdv.advanceExact(localId)) { final BytesRef bRef = sdv.binaryValue(); // Special handling for Boolean fields since they're stored as 'T' and 'F'. if (schemaField.getType() instanceof BoolField) { doc.addField(fieldName, schemaField.getType().toObject(schemaField, bRef)); } else { doc.addField(fieldName, bRef.utf8ToString()); } } break; case SORTED_NUMERIC: final SortedNumericDocValues numericDv = leafReader.getSortedNumericDocValues(fieldName); NumberType type = schemaField.getType().getNumberType(); if (numericDv != null) { if (numericDv.advance(localId) == localId) { final List<Object> outValues = new ArrayList<Object>(numericDv.docValueCount()); for (int i = 0; i < numericDv.docValueCount(); i++) { long number = numericDv.nextValue(); switch (type) { case INTEGER: outValues.add((int) number); break; case LONG: outValues.add(number); break; case FLOAT: outValues.add(NumericUtils.sortableIntToFloat((int) number)); break; case DOUBLE: outValues.add(NumericUtils.sortableLongToDouble(number)); break; case DATE: outValues.add(new Date(number)); break; default: throw new AssertionError("Unexpected PointType: " + type); } } assert outValues.size() > 0; doc.addField(fieldName, outValues); } } case SORTED_SET: final SortedSetDocValues values = leafReader.getSortedSetDocValues(fieldName); if (values != null && values.getValueCount() > 0) { if (values.advance(localId) == localId) { final List<Object> outValues = new LinkedList<>(); for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values .nextOrd()) { value = values.lookupOrd(ord); outValues.add(schemaField.getType().toObject(schemaField, value)); } assert outValues.size() > 0; doc.addField(fieldName, outValues); } } case NONE: break; } } }
From source file:org.apache.solr.uninverting.TestDocTermOrds.java
License:Apache License
private void verify(LeafReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef) throws Exception { final DocTermOrds dto = new DocTermOrds(r, r.getLiveDocs(), "field", prefixRef, Integer.MAX_VALUE, TestUtil.nextInt(random(), 2, 10)); final NumericDocValues docIDToID = FieldCache.DEFAULT.getNumerics(r, "id", FieldCache.LEGACY_INT_PARSER); /*/*from w w w . ja v a 2 s . co m*/ for(int docID=0;docID<subR.maxDoc();docID++) { System.out.println(" docID=" + docID + " id=" + docIDToID[docID]); } */ if (VERBOSE) { System.out.println("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.utf8ToString())); System.out.println("TEST: all TERMS:"); TermsEnum allTE = MultiFields.getTerms(r, "field").iterator(); int ord = 0; while (allTE.next() != null) { System.out.println(" ord=" + (ord++) + " term=" + allTE.term().utf8ToString()); } } //final TermsEnum te = subR.fields().terms("field").iterator(); final TermsEnum te = dto.getOrdTermsEnum(r); if (dto.numTerms() == 0) { if (prefixRef == null) { assertNull(MultiFields.getTerms(r, "field")); } else { Terms terms = MultiFields.getTerms(r, "field"); if (terms != null) { TermsEnum termsEnum = terms.iterator(); TermsEnum.SeekStatus result = termsEnum.seekCeil(prefixRef); if (result != TermsEnum.SeekStatus.END) { assertFalse( "term=" + termsEnum.term().utf8ToString() + " matches prefix=" + prefixRef.utf8ToString(), StringHelper.startsWith(termsEnum.term(), prefixRef)); } else { // ok } } else { // ok } } return; } if (VERBOSE) { System.out.println("TEST: TERMS:"); te.seekExact(0); while (true) { System.out.println(" ord=" + te.ord() + " term=" + te.term().utf8ToString()); if (te.next() == null) { break; } } } SortedSetDocValues iter = dto.iterator(r); for (int docID = 0; docID < r.maxDoc(); docID++) { assertEquals(docID, docIDToID.nextDoc()); if (docID > iter.docID()) { iter.nextDoc(); } if (docID < iter.docID()) { int[] answers = idToOrds[(int) docIDToID.longValue()]; assertEquals(0, answers.length); continue; } if (VERBOSE) { System.out.println( "TEST: docID=" + docID + " of " + r.maxDoc() + " (id=" + docIDToID.longValue() + ")"); } final int[] answers = idToOrds[(int) docIDToID.longValue()]; int upto = 0; long ord; while ((ord = iter.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) { te.seekExact(ord); final BytesRef expected = termsArray[answers[upto++]]; if (VERBOSE) { System.out.println(" exp=" + expected.utf8ToString() + " actual=" + te.term().utf8ToString()); } assertEquals("expected=" + expected.utf8ToString() + " actual=" + te.term().utf8ToString() + " ord=" + ord, expected, te.term()); } assertEquals(answers.length, upto); } }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
public void test() throws IOException { FieldCache cache = FieldCache.DEFAULT; NumericDocValues doubles = cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER); for (int i = 0; i < NUM_DOCS; i++) { assertEquals(i, doubles.nextDoc()); assertEquals(Double.doubleToLongBits(Double.MAX_VALUE - i), doubles.longValue()); }//from ww w . j a v a 2 s.c o m NumericDocValues longs = cache.getNumerics(reader, "theLong", FieldCache.LONG_POINT_PARSER); for (int i = 0; i < NUM_DOCS; i++) { assertEquals(i, longs.nextDoc()); assertEquals(Long.MAX_VALUE - i, longs.longValue()); } NumericDocValues ints = cache.getNumerics(reader, "theInt", FieldCache.INT_POINT_PARSER); for (int i = 0; i < NUM_DOCS; i++) { assertEquals(i, ints.nextDoc()); assertEquals(Integer.MAX_VALUE - i, ints.longValue()); } NumericDocValues floats = cache.getNumerics(reader, "theFloat", FieldCache.FLOAT_POINT_PARSER); for (int i = 0; i < NUM_DOCS; i++) { assertEquals(i, floats.nextDoc()); assertEquals(Float.floatToIntBits(Float.MAX_VALUE - i), floats.longValue()); } Bits docsWithField = cache.getDocsWithField(reader, "theLong", FieldCache.LONG_POINT_PARSER); assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "theLong", FieldCache.LONG_POINT_PARSER)); assertTrue("docsWithField(theLong) must be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits); assertTrue("docsWithField(theLong) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS); for (int i = 0; i < docsWithField.length(); i++) { assertTrue(docsWithField.get(i)); } docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.INT_POINT_PARSER); assertSame("Second request to cache return same array", docsWithField, cache.getDocsWithField(reader, "sparse", FieldCache.INT_POINT_PARSER)); assertFalse("docsWithField(sparse) must not be class Bits.MatchAllBits", docsWithField instanceof Bits.MatchAllBits); assertTrue("docsWithField(sparse) Size: " + docsWithField.length() + " is not: " + NUM_DOCS, docsWithField.length() == NUM_DOCS); for (int i = 0; i < docsWithField.length(); i++) { assertEquals(i % 2 == 0, docsWithField.get(i)); } // getTermsIndex SortedDocValues termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString"); for (int i = 0; i < NUM_DOCS; i++) { final String s; if (i > termsIndex.docID()) { termsIndex.advance(i); } if (i == termsIndex.docID()) { s = termsIndex.binaryValue().utf8ToString(); } else { s = null; } assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s)); } int nTerms = termsIndex.getValueCount(); TermsEnum tenum = termsIndex.termsEnum(); for (int i = 0; i < nTerms; i++) { BytesRef val1 = BytesRef.deepCopyOf(tenum.next()); final BytesRef val = termsIndex.lookupOrd(i); // System.out.println("i="+i); assertEquals(val, val1); } // seek the enum around (note this isn't a great test here) int num = atLeast(100); for (int i = 0; i < num; i++) { int k = random().nextInt(nTerms); final BytesRef val = BytesRef.deepCopyOf(termsIndex.lookupOrd(k)); assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val)); assertEquals(val, tenum.term()); } for (int i = 0; i < nTerms; i++) { final BytesRef val = BytesRef.deepCopyOf(termsIndex.lookupOrd(i)); assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val)); assertEquals(val, tenum.term()); } // test bad field termsIndex = cache.getTermsIndex(reader, "bogusfield"); // getTerms BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString"); for (int i = 0; i < NUM_DOCS; i++) { if (terms.docID() < i) { terms.nextDoc(); } if (terms.docID() == i) { assertEquals(unicodeStrings[i], terms.binaryValue().utf8ToString()); } else { assertNull(unicodeStrings[i]); } } // test bad field terms = cache.getTerms(reader, "bogusfield"); // getDocTermOrds SortedSetDocValues termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField", null); int numEntries = cache.getCacheEntries().length; // ask for it again, and check that we didnt create any additional entries: termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField", null); assertEquals(numEntries, cache.getCacheEntries().length); for (int i = 0; i < NUM_DOCS; i++) { // This will remove identical terms. A DocTermOrds doesn't return duplicate ords for a docId List<BytesRef> values = new ArrayList<>(new LinkedHashSet<>(Arrays.asList(multiValued[i]))); for (BytesRef v : values) { if (v == null) { // why does this test use null values... instead of an empty list: confusing break; } if (i > termOrds.docID()) { assertEquals(i, termOrds.nextDoc()); } long ord = termOrds.nextOrd(); assert ord != SortedSetDocValues.NO_MORE_ORDS; BytesRef scratch = termOrds.lookupOrd(ord); assertEquals(v, scratch); } if (i == termOrds.docID()) { assertEquals(SortedSetDocValues.NO_MORE_ORDS, termOrds.nextOrd()); } } // test bad field termOrds = cache.getDocTermOrds(reader, "bogusfield", null); assertTrue(termOrds.getValueCount() == 0); FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey()); }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
public void testDocsWithField() throws Exception { FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches();//from w w w. ja v a 2 s . c o m assertEquals(0, cache.getCacheEntries().length); cache.getNumerics(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER); // The double[] takes one slots, and docsWithField should also // have been populated: assertEquals(2, cache.getCacheEntries().length); Bits bits = cache.getDocsWithField(reader, "theDouble", FieldCache.DOUBLE_POINT_PARSER); // No new entries should appear: assertEquals(2, cache.getCacheEntries().length); assertTrue(bits instanceof Bits.MatchAllBits); NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER); assertEquals(4, cache.getCacheEntries().length); for (int i = 0; i < reader.maxDoc(); i++) { if (i % 2 == 0) { assertEquals(i, ints.nextDoc()); assertEquals(i, ints.longValue()); } } NumericDocValues numInts = cache.getNumerics(reader, "numInt", FieldCache.INT_POINT_PARSER); for (int i = 0; i < reader.maxDoc(); i++) { if (i % 2 == 0) { assertEquals(i, numInts.nextDoc()); assertEquals(i, numInts.longValue()); } } }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
public void testGetDocsWithFieldThreadSafety() throws Exception { final FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches();/*from w w w . j av a 2s .c o m*/ int NUM_THREADS = 3; Thread[] threads = new Thread[NUM_THREADS]; final AtomicBoolean failed = new AtomicBoolean(); final AtomicInteger iters = new AtomicInteger(); final int NUM_ITER = 200 * RANDOM_MULTIPLIER; final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS, new Runnable() { @Override public void run() { cache.purgeAllCaches(); iters.incrementAndGet(); } }); for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++) { threads[threadIDX] = new Thread() { @Override public void run() { try { while (!failed.get()) { final int op = random().nextInt(3); if (op == 0) { // Purge all caches & resume, once all // threads get here: restart.await(); if (iters.get() >= NUM_ITER) { break; } } else if (op == 1) { Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.INT_POINT_PARSER); for (int i = 0; i < docsWithField.length(); i++) { assertEquals(i % 2 == 0, docsWithField.get(i)); } } else { NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER); for (int i = 0; i < reader.maxDoc(); i++) { if (i % 2 == 0) { assertEquals(i, ints.nextDoc()); assertEquals(i, ints.longValue()); } } } } } catch (Throwable t) { failed.set(true); restart.reset(); throw new RuntimeException(t); } } }; threads[threadIDX].start(); } for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++) { threads[threadIDX].join(); } assertFalse(failed.get()); }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
public void testDocValuesIntegration() throws Exception { Directory dir = newDirectory();//from w w w. j a v a 2 s . co m IndexWriterConfig iwc = newIndexWriterConfig(null); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value"))); doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value"))); doc.add(new NumericDocValuesField("numeric", 42)); doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1"))); doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2"))); iw.addDocument(doc); DirectoryReader ir = iw.getReader(); iw.close(); LeafReader ar = getOnlyLeafReader(ir); // Binary type: can be retrieved via getTerms() expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.INT_POINT_PARSER); }); BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary"); assertEquals(0, binary.nextDoc()); final BytesRef term = binary.binaryValue(); assertEquals("binary value", term.utf8ToString()); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getTermsIndex(ar, "binary"); }); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getDocTermOrds(ar, "binary", null); }); expectThrows(IllegalStateException.class, () -> { new DocTermOrds(ar, null, "binary"); }); Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary", null); assertTrue(bits.get(0)); // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds() expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.INT_POINT_PARSER); }); expectThrows(IllegalStateException.class, () -> { new DocTermOrds(ar, null, "sorted"); }); binary = FieldCache.DEFAULT.getTerms(ar, "sorted"); assertEquals(0, binary.nextDoc()); BytesRef scratch = binary.binaryValue(); assertEquals("sorted value", scratch.utf8ToString()); SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted"); assertEquals(0, sorted.nextDoc()); assertEquals(0, sorted.ordValue()); assertEquals(1, sorted.getValueCount()); scratch = sorted.binaryValue(); assertEquals("sorted value", scratch.utf8ToString()); SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted", null); assertEquals(0, sortedSet.nextDoc()); assertEquals(0, sortedSet.nextOrd()); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); assertEquals(1, sortedSet.getValueCount()); bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted", null); assertTrue(bits.get(0)); // Numeric type: can be retrieved via getInts() and so on NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.INT_POINT_PARSER); assertEquals(0, numeric.nextDoc()); assertEquals(42, numeric.longValue()); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getTerms(ar, "numeric"); }); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getTermsIndex(ar, "numeric"); }); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getDocTermOrds(ar, "numeric", null); }); expectThrows(IllegalStateException.class, () -> { new DocTermOrds(ar, null, "numeric"); }); bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric", null); assertTrue(bits.get(0)); // SortedSet type: can be retrieved via getDocTermOrds() expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.INT_POINT_PARSER); }); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getTerms(ar, "sortedset"); }); expectThrows(IllegalStateException.class, () -> { FieldCache.DEFAULT.getTermsIndex(ar, "sortedset"); }); expectThrows(IllegalStateException.class, () -> { new DocTermOrds(ar, null, "sortedset"); }); sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset", null); assertEquals(0, sortedSet.nextDoc()); assertEquals(0, sortedSet.nextOrd()); assertEquals(1, sortedSet.nextOrd()); assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd()); assertEquals(2, sortedSet.getValueCount()); bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset", null); assertTrue(bits.get(0)); ir.close(); dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
public void testLongFieldCache() throws IOException { Directory dir = newDirectory();//from www .j a v a 2 s.c om IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random())); cfg.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); Document doc = new Document(); LongPoint field = new LongPoint("f", 0L); StoredField field2 = new StoredField("f", 0L); doc.add(field); doc.add(field2); final long[] values = new long[TestUtil.nextInt(random(), 1, 10)]; Set<Integer> missing = new HashSet<>(); for (int i = 0; i < values.length; ++i) { final long v; switch (random().nextInt(10)) { case 0: v = Long.MIN_VALUE; break; case 1: v = 0; break; case 2: v = Long.MAX_VALUE; break; default: v = TestUtil.nextLong(random(), -10, 10); break; } values[i] = v; if (v == 0 && random().nextBoolean()) { // missing iw.addDocument(new Document()); missing.add(i); } else { field.setLongValue(v); field2.setLongValue(v); iw.addDocument(doc); } } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); final NumericDocValues longs = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.LONG_POINT_PARSER); for (int i = 0; i < values.length; ++i) { if (missing.contains(i) == false) { assertEquals(i, longs.nextDoc()); assertEquals(values[i], longs.longValue()); } } assertEquals(NO_MORE_DOCS, longs.nextDoc()); reader.close(); iw.close(); dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
public void testIntFieldCache() throws IOException { Directory dir = newDirectory();/*www .ja v a 2s . c om*/ IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random())); cfg.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg); Document doc = new Document(); IntPoint field = new IntPoint("f", 0); doc.add(field); final int[] values = new int[TestUtil.nextInt(random(), 1, 10)]; Set<Integer> missing = new HashSet<>(); for (int i = 0; i < values.length; ++i) { final int v; switch (random().nextInt(10)) { case 0: v = Integer.MIN_VALUE; break; case 1: v = 0; break; case 2: v = Integer.MAX_VALUE; break; default: v = TestUtil.nextInt(random(), -10, 10); break; } values[i] = v; if (v == 0 && random().nextBoolean()) { // missing iw.addDocument(new Document()); missing.add(i); } else { field.setIntValue(v); iw.addDocument(doc); } } iw.forceMerge(1); final DirectoryReader reader = iw.getReader(); final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.INT_POINT_PARSER); for (int i = 0; i < values.length; ++i) { if (missing.contains(i) == false) { assertEquals(i, ints.nextDoc()); assertEquals(values[i], ints.longValue()); } } assertEquals(NO_MORE_DOCS, ints.nextDoc()); reader.close(); iw.close(); dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCacheReopen.java
License:Apache License
public void testFieldCacheReuseAfterReopen() throws Exception { Directory dir = newDirectory();//from ww w .jav a 2s . co m IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(10))); Document doc = new Document(); doc.add(new IntPoint("number", 17)); writer.addDocument(doc); writer.commit(); // Open reader1 DirectoryReader r = DirectoryReader.open(dir); LeafReader r1 = getOnlyLeafReader(r); final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(r1, "number", FieldCache.INT_POINT_PARSER); assertEquals(0, ints.nextDoc()); assertEquals(17, ints.longValue()); // Add new segment writer.addDocument(doc); writer.commit(); // Reopen reader1 --> reader2 DirectoryReader r2 = DirectoryReader.openIfChanged(r); assertNotNull(r2); r.close(); LeafReader sub0 = r2.leaves().get(0).reader(); final NumericDocValues ints2 = FieldCache.DEFAULT.getNumerics(sub0, "number", FieldCache.INT_POINT_PARSER); r2.close(); assertEquals(0, ints2.nextDoc()); assertEquals(17, ints2.longValue()); writer.close(); dir.close(); }
From source file:org.apache.solr.uninverting.TestFieldCacheWithThreads.java
License:Apache License
public void test() throws Exception { Directory dir = newDirectory();// www .jav a 2 s . c o m IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); final List<Long> numbers = new ArrayList<>(); final List<BytesRef> binary = new ArrayList<>(); final List<BytesRef> sorted = new ArrayList<>(); final int numDocs = atLeast(100); for (int i = 0; i < numDocs; i++) { Document d = new Document(); long number = random().nextLong(); d.add(new NumericDocValuesField("number", number)); BytesRef bytes = new BytesRef(TestUtil.randomRealisticUnicodeString(random())); d.add(new BinaryDocValuesField("bytes", bytes)); binary.add(bytes); bytes = new BytesRef(TestUtil.randomRealisticUnicodeString(random())); d.add(new SortedDocValuesField("sorted", bytes)); sorted.add(bytes); w.addDocument(d); numbers.add(number); } w.forceMerge(1); final IndexReader r = DirectoryReader.open(w); w.close(); assertEquals(1, r.leaves().size()); final LeafReader ar = r.leaves().get(0).reader(); int numThreads = TestUtil.nextInt(random(), 2, 5); List<Thread> threads = new ArrayList<>(); final CountDownLatch startingGun = new CountDownLatch(1); for (int t = 0; t < numThreads; t++) { final Random threadRandom = new Random(random().nextLong()); Thread thread = new Thread() { @Override public void run() { try { startingGun.await(); int iters = atLeast(1000); for (int iter = 0; iter < iters; iter++) { int docID = threadRandom.nextInt(numDocs); switch (threadRandom.nextInt(4)) { case 0: { NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.INT_POINT_PARSER); assertEquals(docID, values.advance(docID)); assertEquals(numbers.get(docID).longValue(), values.longValue()); } break; case 1: { NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.LONG_POINT_PARSER); assertEquals(docID, values.advance(docID)); assertEquals(numbers.get(docID).longValue(), values.longValue()); } break; case 2: { NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.FLOAT_POINT_PARSER); assertEquals(docID, values.advance(docID)); assertEquals(numbers.get(docID).longValue(), values.longValue()); } break; case 3: { NumericDocValues values = FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.DOUBLE_POINT_PARSER); assertEquals(docID, values.advance(docID)); assertEquals(numbers.get(docID).longValue(), values.longValue()); } break; } BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes"); assertEquals(docID, bdv.advance(docID)); assertEquals(binary.get(docID), bdv.binaryValue()); SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted"); assertEquals(docID, sdv.advance(docID)); assertEquals(sorted.get(docID), sdv.binaryValue()); } } catch (Exception e) { throw new RuntimeException(e); } } }; thread.start(); threads.add(thread); } startingGun.countDown(); for (Thread thread : threads) { thread.join(); } r.close(); dir.close(); }