List of usage examples for org.apache.lucene.index FieldInfo hasVectors
public boolean hasVectors()
From source file:com.github.hotware.lucene.extension.highlight.BaseObjectFragmentsBuilder.java
License:BEER-WARE LICENSE
private static Field[] getFields(IndexReader reader, int docId, final String fieldName) throws IOException { // according to javadoc, doc.getFields(fieldName) cannot be used with // lazy loaded field??? final List<Field> fields = new ArrayList<>(); reader.document(docId, new StoredFieldVisitor() { @Override// ww w .ja v a2 s. c o m public void stringField(FieldInfo fieldInfo, String value) { FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); fields.add(new Field(fieldInfo.name, value, ft)); } @Override public Status needsField(FieldInfo fieldInfo) { return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO; } }); return fields.toArray(new Field[fields.size()]); }
From source file:com.o19s.solr.swan.highlight.BaseFragmentsBuilder.java
License:Apache License
protected Field[] getFields(IndexReader reader, int docId, final String fieldName) throws IOException { // according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field??? final List<Field> fields = new ArrayList<Field>(); reader.document(docId, new StoredFieldVisitor() { @Override/*from ww w . j a v a 2 s . c o m*/ public void stringField(FieldInfo fieldInfo, String value) { FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); fields.add(new Field(fieldInfo.name, value, ft)); } @Override public Status needsField(FieldInfo fieldInfo) { return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO; } }); return fields.toArray(new Field[fields.size()]); }
From source file:com.vmware.xenon.services.common.FieldInfoCache.java
License:Open Source License
public static boolean equals(FieldInfo a, FieldInfo b) { return a.number == b.number && a.name.equals(b.name) && a.getPointDimensionCount() == b.getPointDimensionCount() && a.getPointNumBytes() == b.getPointNumBytes() && a.getDocValuesGen() == b.getDocValuesGen() && a.getIndexOptions() == b.getIndexOptions() && a.hasPayloads() == b.hasPayloads() && a.hasVectors() == b.hasVectors() && a.omitsNorms() == b.omitsNorms() && a.hasNorms() == b.hasNorms(); }
From source file:com.vmware.xenon.services.common.Lucene60FieldInfosFormatWithCache.java
License:Open Source License
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormatWithCache.CODEC_NAME, Lucene60FieldInfosFormatWithCache.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size());//from w ww. j av a 2s . c o m for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) { bits |= STORE_TERMVECTOR; } if (fi.omitsNorms()) { bits |= OMIT_NORMS; } if (fi.hasPayloads()) { bits |= STORE_PAYLOADS; } output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); int pointDimensionCount = fi.getPointDimensionCount(); output.writeVInt(pointDimensionCount); if (pointDimensionCount != 0) { output.writeVInt(fi.getPointNumBytes()); } } CodecUtil.writeFooter(output); } }
From source file:io.anserini.index.IndexUtils.java
License:Apache License
void printIndexStats() throws IOException { Fields fields = MultiFields.getFields(reader); Terms terms = fields.terms(LuceneDocumentGenerator.FIELD_BODY); System.out.println("Index statistics"); System.out.println("----------------"); System.out.println("documents: " + reader.numDocs()); System.out.println("documents (non-empty): " + reader.getDocCount(LuceneDocumentGenerator.FIELD_BODY)); System.out.println("unique terms: " + terms.size()); System.out.println(/*w w w.j a va2 s . c o m*/ "total terms: " + reader.getSumTotalTermFreq(LuceneDocumentGenerator.FIELD_BODY)); System.out.println("stored fields:"); FieldInfos fieldInfos = MultiFields.getMergedFieldInfos(reader); for (String fd : fields) { FieldInfo fi = fieldInfos.fieldInfo(fd); System.out.println(" " + fd + " (" + "indexOption: " + fi.getIndexOptions() + ", hasVectors: " + fi.hasVectors() + ", hasPayloads: " + fi.hasPayloads() + ")"); } }
From source file:org.apache.blur.utils.ResetableDocumentStoredFieldVisitor.java
License:Apache License
@Override public void stringField(FieldInfo fieldInfo, String value) throws IOException { final FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); ft.setIndexed(fieldInfo.isIndexed()); ft.setOmitNorms(fieldInfo.omitsNorms()); ft.setIndexOptions(fieldInfo.getIndexOptions()); doc.add(new Field(fieldInfo.name, value, ft)); size += _emptyString * 2;/*from w w w . j a va 2 s . c o m*/ size += fieldInfo.name.length() * 2; size += value.length() * 2; }
From source file:org.apache.solr.uninverting.UninvertingReader.java
License:Apache License
/** * Create a new UninvertingReader with the specified mapping * <p>/*w ww .j a v a2 s . co m*/ * Expert: This should almost never be used. Use {@link #wrap(DirectoryReader, Map)} * instead. * * @lucene.internal */ public UninvertingReader(LeafReader in, Map<String, Type> mapping) { super(in); this.mapping = mapping; ArrayList<FieldInfo> filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { DocValuesType type = fi.getDocValuesType(); if (type == DocValuesType.NONE) { Type t = mapping.get(fi.name); if (t != null) { if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) { // type uses points if (fi.getPointDimensionCount() == 0) { continue; } } else { // type uses inverted index if (fi.getIndexOptions() == IndexOptions.NONE) { continue; } } switch (t) { case INTEGER_POINT: case LONG_POINT: case FLOAT_POINT: case DOUBLE_POINT: case LEGACY_INTEGER: case LEGACY_LONG: case LEGACY_FLOAT: case LEGACY_DOUBLE: type = DocValuesType.NUMERIC; break; case BINARY: type = DocValuesType.BINARY; break; case SORTED: type = DocValuesType.SORTED; break; case SORTED_SET_BINARY: case SORTED_SET_INTEGER: case SORTED_SET_FLOAT: case SORTED_SET_LONG: case SORTED_SET_DOUBLE: type = DocValuesType.SORTED_SET; break; default: throw new AssertionError(); } } } filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(), fi.getPointDimensionCount(), fi.getPointNumBytes())); } fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()])); }
From source file:org.elasticsearch.index.merge.policy.ElasticsearchMergePolicy.java
License:Apache License
/** Return an "upgraded" view of the reader. */ static AtomicReader filter(AtomicReader reader) throws IOException { final FieldInfos fieldInfos = reader.getFieldInfos(); final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME); if (versionInfo != null && versionInfo.hasDocValues()) { // the reader is a recent one, it has versions and they are stored // in a numeric doc values field return reader; }//from w w w .ja va 2s. c om // The segment is an old one, load all versions in memory and hide // them behind a numeric doc values field final Terms terms = reader.terms(UidFieldMapper.NAME); if (terms == null || !terms.hasPayloads()) { // The segment doesn't have an _uid field or doesn't have paylods // don't try to do anything clever. If any other segment has versions // all versions of this segment will be initialized to 0 return reader; } final TermsEnum uids = terms.iterator(null); final GrowableWriter versions = new GrowableWriter(2, reader.maxDoc(), PackedInts.DEFAULT); DocsAndPositionsEnum dpe = null; for (BytesRef uid = uids.next(); uid != null; uid = uids.next()) { dpe = uids.docsAndPositions(reader.getLiveDocs(), dpe, DocsAndPositionsEnum.FLAG_PAYLOADS); assert dpe != null : "field has payloads"; for (int doc = dpe.nextDoc(); doc != DocsEnum.NO_MORE_DOCS; doc = dpe.nextDoc()) { dpe.nextPosition(); final BytesRef payload = dpe.getPayload(); if (payload != null && payload.length == 8) { final long version = Numbers.bytesToLong(payload); versions.set(doc, version); break; } } } // Build new field infos, doc values, and return a filter reader final FieldInfo newVersionInfo; if (versionInfo == null) { // Find a free field number int fieldNumber = 0; for (FieldInfo fi : fieldInfos) { fieldNumber = Math.max(fieldNumber, fi.number + 1); } newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, false, fieldNumber, false, true, false, IndexOptions.DOCS_ONLY, DocValuesType.NUMERIC, DocValuesType.NUMERIC, -1, Collections.<String, String>emptyMap()); } else { newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, versionInfo.isIndexed(), versionInfo.number, versionInfo.hasVectors(), versionInfo.omitsNorms(), versionInfo.hasPayloads(), versionInfo.getIndexOptions(), versionInfo.getDocValuesType(), versionInfo.getNormType(), versionInfo.getDocValuesGen(), versionInfo.attributes()); } final ArrayList<FieldInfo> fieldInfoList = new ArrayList<>(); for (FieldInfo info : fieldInfos) { if (info != versionInfo) { fieldInfoList.add(info); } } fieldInfoList.add(newVersionInfo); final FieldInfos newFieldInfos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()])); final NumericDocValues versionValues = new NumericDocValues() { @Override public long get(int index) { return versions.get(index); } }; return new FilterAtomicReader(reader) { @Override public FieldInfos getFieldInfos() { return newFieldInfos; } @Override public NumericDocValues getNumericDocValues(String field) throws IOException { if (VersionFieldMapper.NAME.equals(field)) { return versionValues; } return super.getNumericDocValues(field); } @Override public Bits getDocsWithField(String field) throws IOException { return new Bits.MatchAllBits(in.maxDoc()); } }; }
From source file:org.elasticsearch.index.merge.policy.IndexUpgraderMergePolicy.java
License:Apache License
/** Return an "upgraded" view of the reader. */ static AtomicReader filter(AtomicReader reader) throws IOException { final FieldInfos fieldInfos = reader.getFieldInfos(); final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME); if (versionInfo != null && versionInfo.hasDocValues()) { // the reader is a recent one, it has versions and they are stored // in a numeric doc values field return reader; }//from ww w .j a v a2 s .c o m // The segment is an old one, load all versions in memory and hide // them behind a numeric doc values field final Terms terms = reader.terms(UidFieldMapper.NAME); if (terms == null || !terms.hasPayloads()) { // The segment doesn't have an _uid field or doesn't have paylods // don't try to do anything clever. If any other segment has versions // all versions of this segment will be initialized to 0 return reader; } final TermsEnum uids = terms.iterator(null); final GrowableWriter versions = new GrowableWriter(2, reader.maxDoc(), PackedInts.DEFAULT); DocsAndPositionsEnum dpe = null; for (BytesRef uid = uids.next(); uid != null; uid = uids.next()) { dpe = uids.docsAndPositions(reader.getLiveDocs(), dpe, DocsAndPositionsEnum.FLAG_PAYLOADS); assert dpe != null : "field has payloads"; for (int doc = dpe.nextDoc(); doc != DocsEnum.NO_MORE_DOCS; doc = dpe.nextDoc()) { dpe.nextPosition(); final BytesRef payload = dpe.getPayload(); if (payload != null && payload.length == 8) { final long version = Numbers.bytesToLong(payload); versions.set(doc, version); break; } } } // Build new field infos, doc values, and return a filter reader final FieldInfo newVersionInfo; if (versionInfo == null) { // Find a free field number int fieldNumber = 0; for (FieldInfo fi : fieldInfos) { fieldNumber = Math.max(fieldNumber, fi.number + 1); } newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, false, fieldNumber, false, true, false, IndexOptions.DOCS_ONLY, DocValuesType.NUMERIC, DocValuesType.NUMERIC, Collections.<String, String>emptyMap()); } else { newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, versionInfo.isIndexed(), versionInfo.number, versionInfo.hasVectors(), versionInfo.omitsNorms(), versionInfo.hasPayloads(), versionInfo.getIndexOptions(), versionInfo.getDocValuesType(), versionInfo.getNormType(), versionInfo.attributes()); } final ArrayList<FieldInfo> fieldInfoList = new ArrayList<FieldInfo>(); for (FieldInfo info : fieldInfos) { if (info != versionInfo) { fieldInfoList.add(info); } } fieldInfoList.add(newVersionInfo); final FieldInfos newFieldInfos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()])); final NumericDocValues versionValues = new NumericDocValues() { @Override public long get(int index) { return versions.get(index); } }; return new FilterAtomicReader(reader) { @Override public FieldInfos getFieldInfos() { return newFieldInfos; } @Override public NumericDocValues getNumericDocValues(String field) throws IOException { if (VersionFieldMapper.NAME.equals(field)) { return versionValues; } return super.getNumericDocValues(field); } @Override public Bits getDocsWithField(String field) throws IOException { return new Bits.MatchAllBits(in.maxDoc()); } }; }
From source file:org.hibernate.search.backend.lucene.search.query.impl.ReusableDocumentStoredFieldVisitor.java
License:LGPL
@Override public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { final FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); ft.setOmitNorms(fieldInfo.omitsNorms()); ft.setIndexOptions(fieldInfo.getIndexOptions()); getDocument().add(new Field(fieldInfo.name, new String(value, StandardCharsets.UTF_8), ft)); }