List of usage examples for org.apache.lucene.index FieldInfos FieldInfos
public FieldInfos(FieldInfo[] infos)
From source file:com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.java
License:Open Source License
DlsFlsFilterLeafReader(final LeafReader delegate, final Set<String> includes, final Query dlsQuery) { super(delegate); flsEnabled = includes != null && !includes.isEmpty(); dlsEnabled = dlsQuery != null;/*from w w w. j a v a2 s.com*/ if (flsEnabled) { this.includes = includes.toArray(new String[0]); final FieldInfos infos = delegate.getFieldInfos(); final List<FieldInfo> fi = new ArrayList<FieldInfo>(infos.size()); for (final FieldInfo info : infos) { final String fname = info.name; if ((!WildcardMatcher.containsWildcard(fname) && includes.contains(fname)) || WildcardMatcher.matchAny(this.includes, fname)) { fi.add(info); } } this.flsFieldInfos = new FieldInfos(fi.toArray(new FieldInfo[0])); } else { this.includes = null; this.flsFieldInfos = null; } if (dlsEnabled) { try { //borrowed from Apache Lucene (Copyright Apache Software Foundation (ASF)) final IndexSearcher searcher = new IndexSearcher(this); searcher.setQueryCache(null); final boolean needsScores = false; final Weight preserveWeight = searcher.createNormalizedWeight(dlsQuery, needsScores); final int maxDoc = in.maxDoc(); final FixedBitSet bits = new FixedBitSet(maxDoc); final Scorer preverveScorer = preserveWeight.scorer(this.getContext()); if (preverveScorer != null) { bits.or(preverveScorer.iterator()); } if (in.hasDeletions()) { final Bits oldLiveDocs = in.getLiveDocs(); assert oldLiveDocs != null; final DocIdSetIterator it = new BitSetIterator(bits, 0L); for (int i = it.nextDoc(); i != DocIdSetIterator.NO_MORE_DOCS; i = it.nextDoc()) { if (!oldLiveDocs.get(i)) { bits.clear(i); } } } this.liveDocs = bits; this.numDocs = bits.cardinality(); } catch (Exception e) { throw new RuntimeException(e); } } else { this.liveDocs = null; this.numDocs = -1; } }
From source file:com.floragunn.searchguard.configuration.EmptyReader.java
License:Apache License
@Override public FieldInfos getFieldInfos() { return new FieldInfos(new FieldInfo[0]); }
From source file:com.vmware.xenon.services.common.FieldInfoCache.java
License:Open Source License
/** * At the end there will be a single segment with all the fields. So it makes sense to cache the longest * list of infos every encountered./*from ww w. j a v a2 s. c o m*/ * * @param infos * @return */ public FieldInfos dedupFieldInfos(FieldInfo[] infos) { FieldInfos cached = this.longest; if (cached == null || cached.size() < infos.length) { cached = new FieldInfos(infos); trimFieldInfos(cached); this.longest = cached; return cached; } if (cached.size() == infos.length) { for (FieldInfo a : infos) { FieldInfo b = cached.fieldInfo(a.number); if (b == null || !FieldInfoCache.equals(a, b)) { FieldInfos update = new FieldInfos(infos); trimFieldInfos(update); this.longest = update; return update; } } return cached; } FieldInfos update = new FieldInfos(infos); trimFieldInfos(update); return update; }
From source file:com.vmware.xenon.services.common.Lucene60FieldInfosFormatWithCache.java
License:Open Source License
@Override public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException { ////////////////////// boolean checkInfosCache = true; ////////////////////// final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) { Throwable priorE = null;//from www.ja v a 2 s . c om FieldInfo[] infos = null; try { CodecUtil.checkIndexHeader(input, Lucene60FieldInfosFormatWithCache.CODEC_NAME, Lucene60FieldInfosFormatWithCache.FORMAT_START, Lucene60FieldInfosFormatWithCache.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); final int size = input.readVInt(); //read in the size infos = new FieldInfo[size]; // previous field's attribute map, we share when possible: Map<String, String> lastAttributes = Collections.emptyMap(); for (int i = 0; i < size; i++) { String name = input.readString(); final int fieldNumber = input.readVInt(); if (fieldNumber < 0) { throw new CorruptIndexException( "invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input); } byte bits = input.readByte(); boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; boolean omitNorms = (bits & OMIT_NORMS) != 0; boolean storePayloads = (bits & STORE_PAYLOADS) != 0; final IndexOptions indexOptions = getIndexOptions(input, input.readByte()); // DV Types are packed in one byte final DocValuesType docValuesType = getDocValuesType(input, input.readByte()); final long dvGen = input.readLong(); Map<String, String> attributes = input.readMapOfStrings(); // just use the last field's map if its the same if (attributes.equals(lastAttributes)) { attributes = lastAttributes; } lastAttributes = attributes; int pointDimensionCount = input.readVInt(); int pointNumBytes; if (pointDimensionCount != 0) { pointNumBytes = input.readVInt(); } else { pointNumBytes = 0; } try { ////////////////////// if (dvGen >= 0) { // skip fields with docValues, they don't cache well checkInfosCache = false; infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, pointDimensionCount, pointNumBytes); } else { infos[i] = this.cache.dedupFieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, pointDimensionCount, pointNumBytes); } ////////////////////// } catch (IllegalStateException e) { throw new CorruptIndexException( "invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e); } } } catch (Throwable exception) { priorE = exception; } finally { CodecUtil.checkFooter(input, priorE); } ////////////////////// if (checkInfosCache) { return this.cache.dedupFieldInfos(infos); } else { FieldInfos result = new FieldInfos(infos); this.cache.trimFieldInfos(result); return result; } ////////////////////// } }
From source file:com.vmware.xenon.services.common.TestFieldInfoCache.java
License:Open Source License
@Test public void testDense() throws NoSuchFieldException, IllegalAccessException { FieldInfo[] infosArray = makeInfosArray(100); FieldInfos orig = new FieldInfos(infosArray); Collection<Object> values = extractValues(orig); assertSame(Collections.unmodifiableCollection(new TreeMap<String, Object>().values()).getClass(), values.getClass());//ww w . jav a2s . c o m FieldInfoCache cache = new FieldInfoCache(); FieldInfos fixedFieldInfo = cache.dedupFieldInfos(infosArray); values = extractValues(fixedFieldInfo); assertSame(Collections.unmodifiableList(new ArrayList<>()).getClass(), values.getClass()); }
From source file:com.vmware.xenon.services.common.TestFieldInfoCache.java
License:Open Source License
@Test public void testSparse() throws NoSuchFieldException, IllegalAccessException { FieldInfo[] infosArray = new FieldInfo[3]; infosArray[0] = makeFieldInfo(1);// w w w.j av a 2 s. c o m infosArray[1] = makeFieldInfo(1000); infosArray[2] = makeFieldInfo(5000); FieldInfos orig = new FieldInfos(infosArray); Collection<Object> values = extractValues(orig); assertSame(Collections.unmodifiableCollection(new TreeMap<String, Object>().values()).getClass(), values.getClass()); FieldInfoCache cache = new FieldInfoCache(); FieldInfos fixedFieldInfo = cache.dedupFieldInfos(infosArray); values = extractValues(fixedFieldInfo); assertSame(Collections.unmodifiableCollection(new TreeMap<String, Object>().values()).getClass(), values.getClass()); }
From source file:dk.statsbiblioteket.netark.dvenabler.wrapper.DVAtomicReader.java
License:Apache License
@Override public FieldInfos getFieldInfos() { log.info("Merging getFieldInfos called with " + maxDoc() + " docs"); long startTime = System.nanoTime(); FieldInfos original = super.getFieldInfos(); FieldInfo[] modified = new FieldInfo[original.size()]; int index = 0; for (FieldInfo oInfo : original) { modified[index++] = dvConfigs.containsKey(oInfo.name) ? dvConfigs.get(oInfo.name).getFieldInfo() : oInfo;/*from ww w . ja va 2 s . c o m*/ } /*FieldInfo mInfo = new FieldInfo( oInfo.name, oInfo.isIndexed(), oInfo.number, oInfo.hasVectors(), oInfo.omitsNorms(), oInfo.hasPayloads(), oInfo.getIndexOptions(), mDocValuesType, oInfo.getNormType(), oInfo.attributes()); */ log.info("Merged " + original.size() + " original and " + dvConfigs.size() + " tweaked FieldInfos for " + maxDoc() + " docs in " + (System.nanoTime() - startTime) / 1000000 + "ms"); return new FieldInfos(modified); }
From source file:org.apache.solr.search.TestDocSet.java
License:Apache License
/**** needs code insertion into HashDocSet public void testExistsCollisions() {//from www. j av a 2 s .c o m loadfactor=.75f; rand=new Random(12345); // make deterministic int maxSetsize=4000; int nSets=512; int[] maxDocs=new int[] {100000,500000,1000000,5000000,10000000}; int ret=0; for (int maxDoc : maxDocs) { int mask = (BitUtil.nextHighestPowerOfTwo(maxDoc)>>1)-1; DocSet[] sets = getRandomHashSets(nSets,maxSetsize, maxDoc); int cstart = HashDocSet.collisions; for (DocSet s1 : sets) { for (int j=0; j<maxDocs[0]; j++) { int idx = rand.nextInt()&mask; ret += s1.exists(idx) ? 1 :0; } } int cend = HashDocSet.collisions; System.out.println("maxDoc="+maxDoc+"\tcollisions="+(cend-cstart)); } if (ret==-1)System.out.println("wow!"); System.out.println("collisions="+HashDocSet.collisions); } ***/ public AtomicReader dummyIndexReader(final int maxDoc) { return new AtomicReader() { @Override public int maxDoc() { return maxDoc; } @Override public int numDocs() { return maxDoc; } @Override public FieldInfos getFieldInfos() { return new FieldInfos(new FieldInfo[0]); } @Override public Bits getLiveDocs() { return null; } @Override public Fields fields() { return null; } @Override public Fields getTermVectors(int doc) { return null; } @Override public NumericDocValues getNumericDocValues(String field) { return null; } @Override public BinaryDocValues getBinaryDocValues(String field) { return null; } @Override public SortedDocValues getSortedDocValues(String field) { return null; } @Override public SortedSetDocValues getSortedSetDocValues(String field) { return null; } @Override public Bits getDocsWithField(String field) throws IOException { return null; } @Override public NumericDocValues getNormValues(String field) { return null; } @Override protected void doClose() { } @Override public void document(int doc, StoredFieldVisitor visitor) { } }; }
From source file:org.apache.solr.uninverting.UninvertingReader.java
License:Apache License
/** * Create a new UninvertingReader with the specified mapping * <p>// www. j a va 2 s . c om * Expert: This should almost never be used. Use {@link #wrap(DirectoryReader, Map)} * instead. * * @lucene.internal */ public UninvertingReader(LeafReader in, Map<String, Type> mapping) { super(in); this.mapping = mapping; ArrayList<FieldInfo> filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { DocValuesType type = fi.getDocValuesType(); if (type == DocValuesType.NONE) { Type t = mapping.get(fi.name); if (t != null) { if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) { // type uses points if (fi.getPointDimensionCount() == 0) { continue; } } else { // type uses inverted index if (fi.getIndexOptions() == IndexOptions.NONE) { continue; } } switch (t) { case INTEGER_POINT: case LONG_POINT: case FLOAT_POINT: case DOUBLE_POINT: case LEGACY_INTEGER: case LEGACY_LONG: case LEGACY_FLOAT: case LEGACY_DOUBLE: type = DocValuesType.NUMERIC; break; case BINARY: type = DocValuesType.BINARY; break; case SORTED: type = DocValuesType.SORTED; break; case SORTED_SET_BINARY: case SORTED_SET_INTEGER: case SORTED_SET_FLOAT: case SORTED_SET_LONG: case SORTED_SET_DOUBLE: type = DocValuesType.SORTED_SET; break; default: throw new AssertionError(); } } } filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(), fi.getPointDimensionCount(), fi.getPointNumBytes())); } fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()])); }
From source file:org.elasticsearch.index.merge.policy.ElasticsearchMergePolicy.java
License:Apache License
/** Return an "upgraded" view of the reader. */ static AtomicReader filter(AtomicReader reader) throws IOException { final FieldInfos fieldInfos = reader.getFieldInfos(); final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME); if (versionInfo != null && versionInfo.hasDocValues()) { // the reader is a recent one, it has versions and they are stored // in a numeric doc values field return reader; }/* w w w. j a v a 2 s .c o m*/ // The segment is an old one, load all versions in memory and hide // them behind a numeric doc values field final Terms terms = reader.terms(UidFieldMapper.NAME); if (terms == null || !terms.hasPayloads()) { // The segment doesn't have an _uid field or doesn't have paylods // don't try to do anything clever. If any other segment has versions // all versions of this segment will be initialized to 0 return reader; } final TermsEnum uids = terms.iterator(null); final GrowableWriter versions = new GrowableWriter(2, reader.maxDoc(), PackedInts.DEFAULT); DocsAndPositionsEnum dpe = null; for (BytesRef uid = uids.next(); uid != null; uid = uids.next()) { dpe = uids.docsAndPositions(reader.getLiveDocs(), dpe, DocsAndPositionsEnum.FLAG_PAYLOADS); assert dpe != null : "field has payloads"; for (int doc = dpe.nextDoc(); doc != DocsEnum.NO_MORE_DOCS; doc = dpe.nextDoc()) { dpe.nextPosition(); final BytesRef payload = dpe.getPayload(); if (payload != null && payload.length == 8) { final long version = Numbers.bytesToLong(payload); versions.set(doc, version); break; } } } // Build new field infos, doc values, and return a filter reader final FieldInfo newVersionInfo; if (versionInfo == null) { // Find a free field number int fieldNumber = 0; for (FieldInfo fi : fieldInfos) { fieldNumber = Math.max(fieldNumber, fi.number + 1); } newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, false, fieldNumber, false, true, false, IndexOptions.DOCS_ONLY, DocValuesType.NUMERIC, DocValuesType.NUMERIC, -1, Collections.<String, String>emptyMap()); } else { newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, versionInfo.isIndexed(), versionInfo.number, versionInfo.hasVectors(), versionInfo.omitsNorms(), versionInfo.hasPayloads(), versionInfo.getIndexOptions(), versionInfo.getDocValuesType(), versionInfo.getNormType(), versionInfo.getDocValuesGen(), versionInfo.attributes()); } final ArrayList<FieldInfo> fieldInfoList = new ArrayList<>(); for (FieldInfo info : fieldInfos) { if (info != versionInfo) { fieldInfoList.add(info); } } fieldInfoList.add(newVersionInfo); final FieldInfos newFieldInfos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()])); final NumericDocValues versionValues = new NumericDocValues() { @Override public long get(int index) { return versions.get(index); } }; return new FilterAtomicReader(reader) { @Override public FieldInfos getFieldInfos() { return newFieldInfos; } @Override public NumericDocValues getNumericDocValues(String field) throws IOException { if (VersionFieldMapper.NAME.equals(field)) { return versionValues; } return super.getNumericDocValues(field); } @Override public Bits getDocsWithField(String field) throws IOException { return new Bits.MatchAllBits(in.maxDoc()); } }; }