Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.index; import java.util.Random; import java.util.function.IntUnaryOperator; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.SortedDocValues; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; import org.apache.solr.util.TestHarness; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; public class UninvertDocValuesMergePolicyTest extends SolrTestCaseJ4 { private static String SOLR_TESTS_SKIP_INTEGRITY_CHECK = "solr.tests.skipIntegrityCheck"; private static String ID_FIELD = "id"; private static String TEST_FIELD = "string_add_dv_later"; @BeforeClass public static void beforeTests() throws Exception { System.setProperty(SOLR_TESTS_SKIP_INTEGRITY_CHECK, (random().nextBoolean() ? "true" : "false")); } @AfterClass public static void afterTests() { System.clearProperty(SOLR_TESTS_SKIP_INTEGRITY_CHECK); } @After public void after() throws Exception { deleteCore(); } @Before public void before() throws Exception { initCore("solrconfig-uninvertdocvaluesmergepolicyfactory.xml", "schema-docValues.xml"); } public void testIndexAndAddDocValues() throws Exception { Random rand = random(); for (int i = 0; i < 100; i++) { assertU(adoc(ID_FIELD, String.valueOf(i), TEST_FIELD, String.valueOf(i))); if (rand.nextBoolean()) { assertU(commit()); } } assertU(commit()); // Assert everything has been indexed and there are no docvalues withNewRawReader(h, topReader -> { assertEquals(100, topReader.numDocs()); final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader); // The global field type should not have docValues yet assertEquals(DocValuesType.NONE, infos.fieldInfo(TEST_FIELD).getDocValuesType()); }); addDocValuesTo(h, TEST_FIELD); // Add some more documents with doc values turned on including updating some for (int i = 90; i < 110; i++) { assertU(adoc(ID_FIELD, String.valueOf(i), TEST_FIELD, String.valueOf(i))); if (rand.nextBoolean()) { assertU(commit()); } } assertU(commit()); withNewRawReader(h, topReader -> { assertEquals(110, topReader.numDocs()); final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader); // The global field type should have docValues because a document with dvs was added assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType()); }); int optimizeSegments = 1; assertU(optimize("maxSegments", String.valueOf(optimizeSegments))); // Assert all docs have the right docvalues withNewRawReader(h, topReader -> { // Assert merged into one segment assertEquals(110, topReader.numDocs()); assertEquals(optimizeSegments, topReader.leaves().size()); final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader); // The global field type should have docValues because a document with dvs was added assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType()); // Check that all segments have the right docvalues type with the correct value // Also check that other fields (e.g. the id field) didn't mistakenly get docvalues added for (LeafReaderContext ctx : topReader.leaves()) { LeafReader r = ctx.reader(); SortedDocValues docvalues = r.getSortedDocValues(TEST_FIELD); for (int i = 0; i < r.numDocs(); ++i) { Document doc = r.document(i); String v = doc.getField(TEST_FIELD).stringValue(); String id = doc.getField(ID_FIELD).stringValue(); assertEquals(DocValuesType.SORTED, r.getFieldInfos().fieldInfo(TEST_FIELD).getDocValuesType()); assertEquals(DocValuesType.NONE, r.getFieldInfos().fieldInfo(ID_FIELD).getDocValuesType()); assertEquals(v, id); docvalues.nextDoc(); assertEquals(v, docvalues.binaryValue().utf8ToString()); } } }); } // When an non-indexed field gets merged, it exhibit the old behavior // The field will be merged, docvalues headers updated, but no docvalues for this field public void testNonIndexedFieldDoesNonFail() throws Exception { // Remove Indexed from fieldType removeIndexFrom(h, TEST_FIELD); assertU(adoc(ID_FIELD, String.valueOf(1), TEST_FIELD, String.valueOf(1))); assertU(commit()); addDocValuesTo(h, TEST_FIELD); assertU(adoc(ID_FIELD, String.valueOf(2), TEST_FIELD, String.valueOf(2))); assertU(commit()); assertU(optimize("maxSegments", "1")); withNewRawReader(h, topReader -> { // Assert merged into one segment assertEquals(2, topReader.numDocs()); assertEquals(1, topReader.leaves().size()); final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader); // The global field type should have docValues because a document with dvs was added assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType()); for (LeafReaderContext ctx : topReader.leaves()) { LeafReader r = ctx.reader(); SortedDocValues docvalues = r.getSortedDocValues(TEST_FIELD); for (int i = 0; i < r.numDocs(); ++i) { Document doc = r.document(i); String v = doc.getField(TEST_FIELD).stringValue(); String id = doc.getField(ID_FIELD).stringValue(); assertEquals(DocValuesType.SORTED, r.getFieldInfos().fieldInfo(TEST_FIELD).getDocValuesType()); assertEquals(DocValuesType.NONE, r.getFieldInfos().fieldInfo(ID_FIELD).getDocValuesType()); if (id.equals("2")) { assertTrue(docvalues.advanceExact(i)); assertEquals(v, docvalues.binaryValue().utf8ToString()); } else { assertFalse(docvalues.advanceExact(i)); } } } }); } private static void addDocValuesTo(TestHarness h, String fieldName) { implUpdateSchemaField(h, fieldName, (p) -> (p | 0x00008000)); // FieldProperties.DOC_VALUES } private static void removeIndexFrom(TestHarness h, String fieldName) { implUpdateSchemaField(h, fieldName, (p) -> (p ^ 0x00000001)); // FieldProperties.INDEXED } private static void implUpdateSchemaField(TestHarness h, String fieldName, IntUnaryOperator propertiesModifier) { try (SolrCore core = h.getCoreInc()) { // Add docvalues to the field type IndexSchema schema = core.getLatestSchema(); SchemaField oldSchemaField = schema.getField(fieldName); SchemaField newSchemaField = new SchemaField(fieldName, oldSchemaField.getType(), propertiesModifier.applyAsInt(oldSchemaField.getProperties()), oldSchemaField.getDefaultValue()); schema.getFields().put(fieldName, newSchemaField); } } private interface DirectoryReaderConsumer { public void accept(DirectoryReader consumer) throws Exception; } private static void withNewRawReader(TestHarness h, DirectoryReaderConsumer consumer) { try (SolrCore core = h.getCoreInc()) { final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true); final SolrIndexSearcher searcher = searcherRef.get(); try { try { consumer.accept(searcher.getRawReader()); } catch (Exception e) { fail(e.toString()); } } finally { searcherRef.decref(); } } } }