List of usage examples for org.apache.lucene.document FloatDocValuesField FloatDocValuesField
public FloatDocValuesField(String name, float value)
From source file:cn.codepub.redis.directory.Main.java
License:Apache License
private static Document addDocument(int i) { Document document = new Document(); document.add(new StringField("key1", "key" + i, Field.Store.YES)); document.add(new IntField("key2", i * 100000, Field.Store.YES)); document.add(new FloatField("key3", (float) i * 100000, Field.Store.YES)); document.add(new LongField("key4", (long) i * 100000, Field.Store.YES)); document.add(new DoubleField("key5", (double) i * 100000, Field.Store.YES)); document.add(new TextField("key6", RandomStringUtils.randomAlphabetic(10), Field.Store.YES)); document.add(new StringField("key7", RandomStringUtils.randomAlphabetic(5), Field.Store.YES)); document.add(new BinaryDocValuesField("key8", new BytesRef(RandomStringUtils.randomAlphabetic(5)))); document.add(new DoubleDocValuesField("key9", RandomUtils.nextDouble(0, 1000))); document.add(new FloatDocValuesField("key10", RandomUtils.nextFloat(0, 1000))); document.add(new LongField("key11", (long) i * 50000, Field.Store.YES)); document.add(new IntField("key12", i * 50000, Field.Store.YES)); document.add(new FloatField("key13", (float) i * 50000, Field.Store.YES)); document.add(new DoubleField("key14", (double) i * 50000, Field.Store.YES)); document.add(new StringField("key15", RandomStringUtils.randomAlphabetic(6), Field.Store.YES)); return document; }
From source file:com.b2international.index.lucene.DocValuesFloatIndexField.java
License:Apache License
@Override public NumericDocValuesField toDocValuesField(Float value) { return new FloatDocValuesField(fieldName(), value); }
From source file:com.o19s.es.ltr.logging.LoggingFetchSubPhaseTests.java
License:Apache License
public static Document buildDoc(String text, float value) throws IOException { String id = UUID.randomUUID().toString(); Document d = new Document(); d.add(newStringField("id", id, Field.Store.YES)); d.add(newStringField("text", text, Field.Store.NO)); d.add(new FloatDocValuesField("score", value)); return d;/*from w w w.j a v a 2 s . co m*/ }
From source file:com.qwazr.search.field.FloatDocValuesType.java
License:Apache License
@Override final public void fillValue(final Object value, final FieldConsumer consumer) { if (value instanceof Number) consumer.accept(new FloatDocValuesField(fieldName, ((Number) value).floatValue())); else/*from ww w . jav a 2 s .c o m*/ consumer.accept(new FloatDocValuesField(fieldName, Float.parseFloat(value.toString()))); }
From source file:com.tuplejump.stargate.Fields.java
License:Apache License
private static Field numericDocValuesField(String name, final AbstractType abstractType, final ByteBuffer byteBufferValue) { final String stripedName = striped + name; CQL3Type cqlType = abstractType.asCQL3Type(); if (cqlType == CQL3Type.Native.TIMESTAMP) { Date date = (Date) abstractType.compose(byteBufferValue); return new NumericDocValuesField(stripedName, date.getTime()); }//w ww . j a v a2 s .c o m Number value = (Number) abstractType.compose(byteBufferValue); if (cqlType == CQL3Type.Native.INT || cqlType == CQL3Type.Native.VARINT || cqlType == CQL3Type.Native.BIGINT || cqlType == CQL3Type.Native.COUNTER) { return new NumericDocValuesField(stripedName, value.longValue()); } else if (cqlType == CQL3Type.Native.FLOAT) { return new FloatDocValuesField(stripedName, value.floatValue()); } else if (cqlType == CQL3Type.Native.DECIMAL || cqlType == CQL3Type.Native.DOUBLE) { return new DoubleDocValuesField(stripedName, value.doubleValue()); } else throw new IllegalArgumentException(String.format("Invalid type for numeric doc values <%s>", cqlType)); }
From source file:org.apache.solr.ltr.TestLTRReRankingPipeline.java
License:Apache License
@Ignore @Test//from w w w .j av a 2 s . co m public void testRescorer() throws IOException { final Directory dir = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("id", "0", Field.Store.YES)); doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 1.0f)); w.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); // 1 extra token, but wizard and oz are close; doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 2.0f)); w.addDocument(doc); final IndexReader r = w.getReader(); w.close(); // Do ordinary BooleanQuery: final BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); bqBuilder.add(new TermQuery(new Term("field", "wizard")), BooleanClause.Occur.SHOULD); bqBuilder.add(new TermQuery(new Term("field", "oz")), BooleanClause.Occur.SHOULD); final IndexSearcher searcher = getSearcher(r); // first run the standard query TopDocs hits = searcher.search(bqBuilder.build(), 10); assertEquals(2, hits.totalHits); assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); final List<Feature> features = makeFieldValueFeatures(new int[] { 0, 1, 2 }, "final-score"); final List<Normalizer> norms = new ArrayList<Normalizer>( Collections.nCopies(features.size(), IdentityNormalizer.INSTANCE)); final List<Feature> allFeatures = makeFieldValueFeatures(new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, "final-score"); final LTRScoringModel ltrScoringModel = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, null); final LTRRescorer rescorer = new LTRRescorer(new LTRScoringQuery(ltrScoringModel)); hits = rescorer.rescore(searcher, hits, 2); // rerank using the field final-score assertEquals("1", searcher.doc(hits.scoreDocs[0].doc).get("id")); assertEquals("0", searcher.doc(hits.scoreDocs[1].doc).get("id")); r.close(); dir.close(); }
From source file:org.apache.solr.ltr.TestLTRReRankingPipeline.java
License:Apache License
@Ignore @Test//from ww w . j a v a2 s . com public void testDifferentTopN() throws IOException { final Directory dir = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("id", "0", Field.Store.YES)); doc.add(newTextField("field", "wizard oz oz oz oz oz", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 1.0f)); w.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); doc.add(newTextField("field", "wizard oz oz oz oz the", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 2.0f)); w.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "2", Field.Store.YES)); doc.add(newTextField("field", "wizard oz oz oz the the ", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 3.0f)); w.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "3", Field.Store.YES)); doc.add(newTextField("field", "wizard oz oz the the the the ", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 4.0f)); w.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "4", Field.Store.YES)); doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 5.0f)); w.addDocument(doc); final IndexReader r = w.getReader(); w.close(); // Do ordinary BooleanQuery: final BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); bqBuilder.add(new TermQuery(new Term("field", "wizard")), BooleanClause.Occur.SHOULD); bqBuilder.add(new TermQuery(new Term("field", "oz")), BooleanClause.Occur.SHOULD); final IndexSearcher searcher = getSearcher(r); // first run the standard query TopDocs hits = searcher.search(bqBuilder.build(), 10); assertEquals(5, hits.totalHits); assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); assertEquals("2", searcher.doc(hits.scoreDocs[2].doc).get("id")); assertEquals("3", searcher.doc(hits.scoreDocs[3].doc).get("id")); assertEquals("4", searcher.doc(hits.scoreDocs[4].doc).get("id")); final List<Feature> features = makeFieldValueFeatures(new int[] { 0, 1, 2 }, "final-score"); final List<Normalizer> norms = new ArrayList<Normalizer>( Collections.nCopies(features.size(), IdentityNormalizer.INSTANCE)); final List<Feature> allFeatures = makeFieldValueFeatures(new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, "final-score"); final LTRScoringModel ltrScoringModel = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, null); final LTRRescorer rescorer = new LTRRescorer(new LTRScoringQuery(ltrScoringModel)); // rerank @ 0 should not change the order hits = rescorer.rescore(searcher, hits, 0); assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); assertEquals("2", searcher.doc(hits.scoreDocs[2].doc).get("id")); assertEquals("3", searcher.doc(hits.scoreDocs[3].doc).get("id")); assertEquals("4", searcher.doc(hits.scoreDocs[4].doc).get("id")); // test rerank with different topN cuts for (int topN = 1; topN <= 5; topN++) { log.info("rerank {} documents ", topN); hits = searcher.search(bqBuilder.build(), 10); final ScoreDoc[] slice = new ScoreDoc[topN]; System.arraycopy(hits.scoreDocs, 0, slice, 0, topN); hits = new TopDocs(hits.totalHits, slice, hits.getMaxScore()); hits = rescorer.rescore(searcher, hits, topN); for (int i = topN - 1, j = 0; i >= 0; i--, j++) { log.info("doc {} in pos {}", searcher.doc(hits.scoreDocs[j].doc).get("id"), j); assertEquals(i, Integer.parseInt(searcher.doc(hits.scoreDocs[j].doc).get("id"))); assertEquals(i + 1, hits.scoreDocs[j].score, 0.00001); } } r.close(); dir.close(); }
From source file:org.apache.solr.ltr.TestLTRScoringQuery.java
License:Apache License
@Test public void testLTRScoringQuery() throws IOException, ModelException { final Directory dir = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("id", "0", Field.Store.YES)); doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 1.0f)); w.addDocument(doc);/* ww w . jav a2 s. com*/ doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); // 1 extra token, but wizard and oz are close; doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 2.0f)); w.addDocument(doc); final IndexReader r = w.getReader(); w.close(); // Do ordinary BooleanQuery: final BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); bqBuilder.add(new TermQuery(new Term("field", "wizard")), BooleanClause.Occur.SHOULD); bqBuilder.add(new TermQuery(new Term("field", "oz")), BooleanClause.Occur.SHOULD); final IndexSearcher searcher = getSearcher(r); // first run the standard query final TopDocs hits = searcher.search(bqBuilder.build(), 10); assertEquals(2, hits.totalHits); assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); List<Feature> features = makeFeatures(new int[] { 0, 1, 2 }); final List<Feature> allFeatures = makeFeatures(new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); List<Normalizer> norms = new ArrayList<Normalizer>( Collections.nCopies(features.size(), IdentityNormalizer.INSTANCE)); LTRScoringModel ltrScoringModel = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, makeFeatureWeights(features)); LTRScoringQuery.ModelWeight modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel)); assertEquals(3, modelWeight.getModelFeatureValuesNormalized().length); for (int i = 0; i < 3; i++) { assertEquals(i, modelWeight.getModelFeatureValuesNormalized()[i], 0.0001); } int[] posVals = new int[] { 0, 1, 2 }; int pos = 0; for (LTRScoringQuery.FeatureInfo fInfo : modelWeight.getFeaturesInfo()) { if (fInfo == null) { continue; } assertEquals(posVals[pos], fInfo.getValue(), 0.0001); assertEquals("f" + posVals[pos], fInfo.getName()); pos++; } final int[] mixPositions = new int[] { 8, 2, 4, 9, 0 }; features = makeFeatures(mixPositions); norms = new ArrayList<Normalizer>(Collections.nCopies(features.size(), IdentityNormalizer.INSTANCE)); ltrScoringModel = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, makeFeatureWeights(features)); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel)); assertEquals(mixPositions.length, modelWeight.getModelFeatureWeights().length); for (int i = 0; i < mixPositions.length; i++) { assertEquals(mixPositions[i], modelWeight.getModelFeatureValuesNormalized()[i], 0.0001); } final ModelException expectedModelException = new ModelException("no features declared for model test"); final int[] noPositions = new int[] {}; features = makeFeatures(noPositions); norms = new ArrayList<Normalizer>(Collections.nCopies(features.size(), IdentityNormalizer.INSTANCE)); try { ltrScoringModel = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, makeFeatureWeights(features)); fail("unexpectedly got here instead of catching " + expectedModelException); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel)); assertEquals(0, modelWeight.getModelFeatureWeights().length); } catch (ModelException actualModelException) { assertEquals(expectedModelException.toString(), actualModelException.toString()); } // test normalizers features = makeFilterFeatures(mixPositions); final Normalizer norm = new Normalizer() { @Override public float normalize(float value) { return 42.42f; } @Override public LinkedHashMap<String, Object> paramsToMap() { return null; } @Override protected void validate() throws NormalizerException { } }; norms = new ArrayList<Normalizer>(Collections.nCopies(features.size(), norm)); final LTRScoringModel normMeta = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, makeFeatureWeights(features)); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(normMeta)); normMeta.normalizeFeaturesInPlace(modelWeight.getModelFeatureValuesNormalized()); assertEquals(mixPositions.length, modelWeight.getModelFeatureWeights().length); for (int i = 0; i < mixPositions.length; i++) { assertEquals(42.42f, modelWeight.getModelFeatureValuesNormalized()[i], 0.0001); } r.close(); dir.close(); }
From source file:org.apache.solr.ltr.TestSelectiveWeightCreation.java
License:Apache License
@Test public void testScoringQueryWeightCreation() throws IOException, ModelException { final Directory dir = newDirectory(); final RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("id", "0", Field.Store.YES)); doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 1.0f)); w.addDocument(doc);//from w w w . j av a2 s. c o m doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); // 1 extra token, but wizard and oz are close; doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 2.0f)); w.addDocument(doc); final IndexReader r = w.getReader(); w.close(); // Do ordinary BooleanQuery: final BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder(); bqBuilder.add(new TermQuery(new Term("field", "wizard")), BooleanClause.Occur.SHOULD); bqBuilder.add(new TermQuery(new Term("field", "oz")), BooleanClause.Occur.SHOULD); final IndexSearcher searcher = getSearcher(r); // first run the standard query final TopDocs hits = searcher.search(bqBuilder.build(), 10); assertEquals(2, hits.totalHits); assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); List<Feature> features = makeFeatures(new int[] { 0, 1, 2 }); final List<Feature> allFeatures = makeFeatures(new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); final List<Normalizer> norms = new ArrayList<>(); for (int k = 0; k < features.size(); ++k) { norms.add(IdentityNormalizer.INSTANCE); } // when features are NOT requested in the response, only the modelFeature weights should be created final LTRScoringModel ltrScoringModel1 = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, makeFeatureWeights(features)); LTRScoringQuery.ModelWeight modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel1, false)); // features not requested in response LTRScoringQuery.FeatureInfo[] featuresInfo = modelWeight.getFeaturesInfo(); assertEquals(features.size(), modelWeight.getModelFeatureValuesNormalized().length); int validFeatures = 0; for (int i = 0; i < featuresInfo.length; ++i) { if (featuresInfo[i] != null && featuresInfo[i].isUsed()) { validFeatures += 1; } } assertEquals(validFeatures, features.size()); // when features are requested in the response, weights should be created for all features final LTRScoringModel ltrScoringModel2 = TestLinearModel.createLinearModel("test", features, norms, "test", allFeatures, makeFeatureWeights(features)); modelWeight = performQuery(hits, searcher, hits.scoreDocs[0].doc, new LTRScoringQuery(ltrScoringModel2, true)); // features requested in response featuresInfo = modelWeight.getFeaturesInfo(); assertEquals(features.size(), modelWeight.getModelFeatureValuesNormalized().length); assertEquals(allFeatures.size(), modelWeight.getExtractedFeatureWeights().length); validFeatures = 0; for (int i = 0; i < featuresInfo.length; ++i) { if (featuresInfo[i] != null && featuresInfo[i].isUsed()) { validFeatures += 1; } } assertEquals(validFeatures, allFeatures.size()); assertU(delI("0")); assertU(delI("1")); r.close(); dir.close(); }
From source file:org.codice.ddf.spatial.geocoding.index.GeoNamesLuceneIndexer.java
License:Open Source License
private void addDocument(final IndexWriter indexWriter, final GeoEntry geoEntry, final SpatialStrategy strategy) throws IOException { final Document document = new Document(); document.add(new TextField(GeoNamesLuceneConstants.NAME_FIELD, geoEntry.getName(), Field.Store.YES)); document.add(new StoredField(GeoNamesLuceneConstants.LATITUDE_FIELD, geoEntry.getLatitude())); document.add(new StoredField(GeoNamesLuceneConstants.LONGITUDE_FIELD, geoEntry.getLongitude())); document.add(new StringField(GeoNamesLuceneConstants.FEATURE_CODE_FIELD, geoEntry.getFeatureCode(), Field.Store.YES));/* w w w. ja va 2 s. c o m*/ document.add(new TextField(GeoNamesLuceneConstants.COUNTRY_CODE_FIELD, geoEntry.getCountryCode(), Field.Store.YES)); document.add(new StoredField(GeoNamesLuceneConstants.POPULATION_FIELD, geoEntry.getPopulation())); // This DocValues field is used for sorting by population. document.add(new NumericDocValuesField(GeoNamesLuceneConstants.POPULATION_DOCVALUES_FIELD, geoEntry.getPopulation())); document.add(new TextField(GeoNamesLuceneConstants.ALTERNATE_NAMES_FIELD, geoEntry.getAlternateNames(), Field.Store.NO)); // Add each entry's spatial information for fast spatial filtering. final Shape point = SPATIAL_CONTEXT.getShapeFactory().pointXY(geoEntry.getLongitude(), geoEntry.getLatitude()); for (IndexableField field : strategy.createIndexableFields(point)) { document.add(field); } final float boost = calculateBoost(geoEntry); document.add(new FloatDocValuesField(GeoNamesLuceneConstants.BOOST_FIELD, boost)); indexWriter.addDocument(document); }