List of usage examples for org.apache.lucene.index FieldInfo attributes
Map attributes
To view the source code for org.apache.lucene.index FieldInfo attributes.
Click Source Link
From source file:com.vmware.xenon.services.common.Lucene60FieldInfosFormatWithCache.java
License:Open Source License
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormatWithCache.CODEC_NAME, Lucene60FieldInfosFormatWithCache.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size());/*from w w w . ja va2s . co m*/ for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) { bits |= STORE_TERMVECTOR; } if (fi.omitsNorms()) { bits |= OMIT_NORMS; } if (fi.hasPayloads()) { bits |= STORE_PAYLOADS; } output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); int pointDimensionCount = fi.getPointDimensionCount(); output.writeVInt(pointDimensionCount); if (pointDimensionCount != 0) { output.writeVInt(fi.getPointNumBytes()); } } CodecUtil.writeFooter(output); } }
From source file:org.apache.solr.uninverting.UninvertingReader.java
License:Apache License
/** * Create a new UninvertingReader with the specified mapping * <p>//from ww w. j a v a 2s. c o m * Expert: This should almost never be used. Use {@link #wrap(DirectoryReader, Map)} * instead. * * @lucene.internal */ public UninvertingReader(LeafReader in, Map<String, Type> mapping) { super(in); this.mapping = mapping; ArrayList<FieldInfo> filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { DocValuesType type = fi.getDocValuesType(); if (type == DocValuesType.NONE) { Type t = mapping.get(fi.name); if (t != null) { if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) { // type uses points if (fi.getPointDimensionCount() == 0) { continue; } } else { // type uses inverted index if (fi.getIndexOptions() == IndexOptions.NONE) { continue; } } switch (t) { case INTEGER_POINT: case LONG_POINT: case FLOAT_POINT: case DOUBLE_POINT: case LEGACY_INTEGER: case LEGACY_LONG: case LEGACY_FLOAT: case LEGACY_DOUBLE: type = DocValuesType.NUMERIC; break; case BINARY: type = DocValuesType.BINARY; break; case SORTED: type = DocValuesType.SORTED; break; case SORTED_SET_BINARY: case SORTED_SET_INTEGER: case SORTED_SET_FLOAT: case SORTED_SET_LONG: case SORTED_SET_DOUBLE: type = DocValuesType.SORTED_SET; break; default: throw new AssertionError(); } } } filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(), fi.getPointDimensionCount(), fi.getPointNumBytes())); } fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()])); }
From source file:org.elasticsearch.index.merge.policy.ElasticsearchMergePolicy.java
License:Apache License
/** Return an "upgraded" view of the reader. */ static AtomicReader filter(AtomicReader reader) throws IOException { final FieldInfos fieldInfos = reader.getFieldInfos(); final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME); if (versionInfo != null && versionInfo.hasDocValues()) { // the reader is a recent one, it has versions and they are stored // in a numeric doc values field return reader; }//from w ww . j a v a 2 s . c o m // The segment is an old one, load all versions in memory and hide // them behind a numeric doc values field final Terms terms = reader.terms(UidFieldMapper.NAME); if (terms == null || !terms.hasPayloads()) { // The segment doesn't have an _uid field or doesn't have paylods // don't try to do anything clever. If any other segment has versions // all versions of this segment will be initialized to 0 return reader; } final TermsEnum uids = terms.iterator(null); final GrowableWriter versions = new GrowableWriter(2, reader.maxDoc(), PackedInts.DEFAULT); DocsAndPositionsEnum dpe = null; for (BytesRef uid = uids.next(); uid != null; uid = uids.next()) { dpe = uids.docsAndPositions(reader.getLiveDocs(), dpe, DocsAndPositionsEnum.FLAG_PAYLOADS); assert dpe != null : "field has payloads"; for (int doc = dpe.nextDoc(); doc != DocsEnum.NO_MORE_DOCS; doc = dpe.nextDoc()) { dpe.nextPosition(); final BytesRef payload = dpe.getPayload(); if (payload != null && payload.length == 8) { final long version = Numbers.bytesToLong(payload); versions.set(doc, version); break; } } } // Build new field infos, doc values, and return a filter reader final FieldInfo newVersionInfo; if (versionInfo == null) { // Find a free field number int fieldNumber = 0; for (FieldInfo fi : fieldInfos) { fieldNumber = Math.max(fieldNumber, fi.number + 1); } newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, false, fieldNumber, false, true, false, IndexOptions.DOCS_ONLY, DocValuesType.NUMERIC, DocValuesType.NUMERIC, -1, Collections.<String, String>emptyMap()); } else { newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, versionInfo.isIndexed(), versionInfo.number, versionInfo.hasVectors(), versionInfo.omitsNorms(), versionInfo.hasPayloads(), versionInfo.getIndexOptions(), versionInfo.getDocValuesType(), versionInfo.getNormType(), versionInfo.getDocValuesGen(), versionInfo.attributes()); } final ArrayList<FieldInfo> fieldInfoList = new ArrayList<>(); for (FieldInfo info : fieldInfos) { if (info != versionInfo) { fieldInfoList.add(info); } } fieldInfoList.add(newVersionInfo); final FieldInfos newFieldInfos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()])); final NumericDocValues versionValues = new NumericDocValues() { @Override public long get(int index) { return versions.get(index); } }; return new FilterAtomicReader(reader) { @Override public FieldInfos getFieldInfos() { return newFieldInfos; } @Override public NumericDocValues getNumericDocValues(String field) throws IOException { if (VersionFieldMapper.NAME.equals(field)) { return versionValues; } return super.getNumericDocValues(field); } @Override public Bits getDocsWithField(String field) throws IOException { return new Bits.MatchAllBits(in.maxDoc()); } }; }
From source file:org.elasticsearch.index.merge.policy.IndexUpgraderMergePolicy.java
License:Apache License
/** Return an "upgraded" view of the reader. */ static AtomicReader filter(AtomicReader reader) throws IOException { final FieldInfos fieldInfos = reader.getFieldInfos(); final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME); if (versionInfo != null && versionInfo.hasDocValues()) { // the reader is a recent one, it has versions and they are stored // in a numeric doc values field return reader; }/*from w w w . j a v a2 s . co m*/ // The segment is an old one, load all versions in memory and hide // them behind a numeric doc values field final Terms terms = reader.terms(UidFieldMapper.NAME); if (terms == null || !terms.hasPayloads()) { // The segment doesn't have an _uid field or doesn't have paylods // don't try to do anything clever. If any other segment has versions // all versions of this segment will be initialized to 0 return reader; } final TermsEnum uids = terms.iterator(null); final GrowableWriter versions = new GrowableWriter(2, reader.maxDoc(), PackedInts.DEFAULT); DocsAndPositionsEnum dpe = null; for (BytesRef uid = uids.next(); uid != null; uid = uids.next()) { dpe = uids.docsAndPositions(reader.getLiveDocs(), dpe, DocsAndPositionsEnum.FLAG_PAYLOADS); assert dpe != null : "field has payloads"; for (int doc = dpe.nextDoc(); doc != DocsEnum.NO_MORE_DOCS; doc = dpe.nextDoc()) { dpe.nextPosition(); final BytesRef payload = dpe.getPayload(); if (payload != null && payload.length == 8) { final long version = Numbers.bytesToLong(payload); versions.set(doc, version); break; } } } // Build new field infos, doc values, and return a filter reader final FieldInfo newVersionInfo; if (versionInfo == null) { // Find a free field number int fieldNumber = 0; for (FieldInfo fi : fieldInfos) { fieldNumber = Math.max(fieldNumber, fi.number + 1); } newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, false, fieldNumber, false, true, false, IndexOptions.DOCS_ONLY, DocValuesType.NUMERIC, DocValuesType.NUMERIC, Collections.<String, String>emptyMap()); } else { newVersionInfo = new FieldInfo(VersionFieldMapper.NAME, versionInfo.isIndexed(), versionInfo.number, versionInfo.hasVectors(), versionInfo.omitsNorms(), versionInfo.hasPayloads(), versionInfo.getIndexOptions(), versionInfo.getDocValuesType(), versionInfo.getNormType(), versionInfo.attributes()); } final ArrayList<FieldInfo> fieldInfoList = new ArrayList<FieldInfo>(); for (FieldInfo info : fieldInfos) { if (info != versionInfo) { fieldInfoList.add(info); } } fieldInfoList.add(newVersionInfo); final FieldInfos newFieldInfos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()])); final NumericDocValues versionValues = new NumericDocValues() { @Override public long get(int index) { return versions.get(index); } }; return new FilterAtomicReader(reader) { @Override public FieldInfos getFieldInfos() { return newFieldInfos; } @Override public NumericDocValues getNumericDocValues(String field) throws IOException { if (VersionFieldMapper.NAME.equals(field)) { return versionValues; } return super.getNumericDocValues(field); } @Override public Bits getDocsWithField(String field) throws IOException { return new Bits.MatchAllBits(in.maxDoc()); } }; }
From source file:org.eu.bitzone.Leia.java
License:Apache License
public void showDiagnostics(final Object segmentsTable) { final Object diagsTable = find("diagsTable"); removeAll(diagsTable);// w ww .j ava 2s .c om final Object row = getSelectedItem(segmentsTable); if (row == null) { return; } final SegmentCommitInfo si = (SegmentCommitInfo) getProperty(row, "si"); if (si == null) { showStatus("Missing SegmentInfoPerCommit???"); return; } Map<String, String> map = si.info.attributes(); if (map != null) { for (final Entry<String, String> e : map.entrySet()) { final Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "A"); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey()); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); add(r, cell); } } // separator // Object r1 = create("row"); // add(diagsTable, r1); // Object c1 = create("cell"); // setBoolean(c1, "enabled", false); // add(r1, c1); map = si.info.getDiagnostics(); if (map != null) { for (final Entry<String, String> e : map.entrySet()) { final Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "D"); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey()); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); add(r, cell); } } // separator Object r1 = create("row"); add(diagsTable, r1); Object c1 = create("cell"); setBoolean(c1, "enabled", false); add(r1, c1); // codec info final Codec codec = si.info.getCodec(); map = new LinkedHashMap<String, String>(); map.put("codecName", codec.getName()); map.put("codecClassName", codec.getClass().getName()); map.put("docValuesFormat", codec.docValuesFormat().getClass().getName()); map.put("fieldInfosFormat", codec.fieldInfosFormat().getClass().getName()); map.put("liveDocsFormat", codec.liveDocsFormat().getClass().getName()); map.put("normsFormat", codec.normsFormat().getClass().getName()); map.put("postingsFormat", codec.postingsFormat().toString() + " " + codec.postingsFormat().getClass().getName()); map.put("segmentInfoFormat", codec.segmentInfoFormat().getClass().getName()); map.put("storedFieldsFormat", codec.storedFieldsFormat().getClass().getName()); map.put("termVectorsFormat", codec.termVectorsFormat().getClass().getName()); try { final List<String> files = new ArrayList<String>(si.files()); Collections.sort(files); map.put("---files---", files.toString()); if (si.info.getUseCompoundFile()) { final Directory d = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READ, false); files.clear(); files.addAll(Arrays.asList(d.listAll())); d.close(); Collections.sort(files); map.put("-CFS-files-", files.toString()); } } catch (final Exception e) { e.printStackTrace(); map.put("---files---", "Exception: " + e.toString()); } for (final Entry<String, String> e : map.entrySet()) { final Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "C"); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey()); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); add(r, cell); } // fieldInfos try { final SegmentReader sr = new SegmentReader(si, 1, IOContext.READ); final FieldInfos fis = sr.getFieldInfos(); map = new LinkedHashMap<String, String>(); final List<String> flds = new ArrayList<String>(fis.size()); for (final FieldInfo fi : fis) { flds.add(fi.name); } Collections.sort(flds); map.put("L---fields---", flds.toString()); for (final String fn : flds) { final FieldInfo fi = fis.fieldInfo(fn); map.put("A" + fi.name, fi.attributes().toString()); } map.put("F---flags----", "IdfpoPVNtxxDtxx"); for (final String fn : flds) { final FieldInfo fi = fis.fieldInfo(fn); map.put("F" + fi.name, Util.fieldFlags(null, fi)); } sr.close(); // separator r1 = create("row"); add(diagsTable, r1); c1 = create("cell"); setBoolean(c1, "enabled", false); add(r1, c1); for (final Entry<String, String> e : map.entrySet()) { final Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "F" + e.getKey().charAt(0)); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey().substring(1)); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); if (e.getKey().startsWith("F")) { setFont(cell, courier); } add(r, cell); } } catch (final IOException e1) { e1.printStackTrace(); } }
From source file:org.getopt.luke.Luke.java
License:Apache License
public void showDiagnostics(Object segmentsTable) { Object diagsTable = find("diagsTable"); removeAll(diagsTable);//w w w .java 2s .c o m Object row = getSelectedItem(segmentsTable); if (row == null) { return; } SegmentCommitInfo si = (SegmentCommitInfo) getProperty(row, "si"); if (si == null) { showStatus("Missing SegmentInfoPerCommit???"); return; } Map<String, String> map = si.info.attributes(); if (map != null) { for (Entry<String, String> e : map.entrySet()) { Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "A"); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey()); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); add(r, cell); } } // separator // Object r1 = create("row"); // add(diagsTable, r1); // Object c1 = create("cell"); // setBoolean(c1, "enabled", false); // add(r1, c1); map = si.info.getDiagnostics(); if (map != null) { for (Entry<String, String> e : map.entrySet()) { Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "D"); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey()); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); add(r, cell); } } // separator Object r1 = create("row"); add(diagsTable, r1); Object c1 = create("cell"); setBoolean(c1, "enabled", false); add(r1, c1); // codec info Codec codec = si.info.getCodec(); map = new LinkedHashMap<String, String>(); map.put("codecName", codec.getName()); map.put("codecClassName", codec.getClass().getName()); map.put("docValuesFormat", codec.docValuesFormat().getClass().getName()); map.put("fieldInfosFormat", codec.fieldInfosFormat().getClass().getName()); map.put("liveDocsFormat", codec.liveDocsFormat().getClass().getName()); map.put("normsFormat", codec.normsFormat().getClass().getName()); map.put("postingsFormat", codec.postingsFormat().toString() + " " + codec.postingsFormat().getClass().getName()); map.put("segmentInfoFormat", codec.segmentInfoFormat().getClass().getName()); map.put("storedFieldsFormat", codec.storedFieldsFormat().getClass().getName()); map.put("termVectorsFormat", codec.termVectorsFormat().getClass().getName()); try { List<String> files = new ArrayList<String>(si.files()); Collections.sort(files); map.put("---files---", files.toString()); if (si.info.getUseCompoundFile()) { Directory d = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READ, false); files.clear(); files.addAll(Arrays.asList(d.listAll())); d.close(); Collections.sort(files); map.put("-CFS-files-", files.toString()); } } catch (Exception e) { e.printStackTrace(); map.put("---files---", "Exception: " + e.toString()); } for (Entry<String, String> e : map.entrySet()) { Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "C"); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey()); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); add(r, cell); } // fieldInfos try { SegmentReader sr = new SegmentReader(si, 1, IOContext.READ); FieldInfos fis = sr.getFieldInfos(); map = new LinkedHashMap<String, String>(); List<String> flds = new ArrayList<String>(fis.size()); for (FieldInfo fi : fis) { flds.add(fi.name); } Collections.sort(flds); map.put("L---fields---", flds.toString()); for (String fn : flds) { FieldInfo fi = fis.fieldInfo(fn); map.put("A" + fi.name, fi.attributes().toString()); } map.put("F---flags----", "IdfpoPVNtxxDtxx"); for (String fn : flds) { FieldInfo fi = fis.fieldInfo(fn); map.put("F" + fi.name, Util.fieldFlags(null, fi)); } sr.close(); // separator r1 = create("row"); add(diagsTable, r1); c1 = create("cell"); setBoolean(c1, "enabled", false); add(r1, c1); for (Entry<String, String> e : map.entrySet()) { Object r = create("row"); add(diagsTable, r); Object cell = create("cell"); setString(cell, "text", "F" + e.getKey().charAt(0)); add(r, cell); cell = create("cell"); setString(cell, "text", e.getKey().substring(1)); add(r, cell); cell = create("cell"); setString(cell, "text", e.getValue()); if (e.getKey().startsWith("F")) { setFont(cell, courier); } add(r, cell); } } catch (IOException e1) { e1.printStackTrace(); } }
From source file:perf.DiskUsage.java
License:Apache License
static Set<FieldStats> analyzeFields(SegmentReader reader) throws Exception { Map<String, FieldStats> stats = new HashMap<>(); Map<String, String> dvSuffixes = new HashMap<>(); Map<String, String> postingsSuffixes = new HashMap<>(); for (FieldInfo field : reader.getFieldInfos()) { FieldStats fieldStats = new FieldStats(field.name); stats.put(field.name, fieldStats); Map<String, String> attributes = field.attributes(); if (attributes != null) { String postingsSuffix = attributes.get(PerFieldPostingsFormat.PER_FIELD_SUFFIX_KEY); if (postingsSuffix != null) { postingsSuffixes.put(postingsSuffix, field.name); }/*www . ja va 2 s .c om*/ String dvSuffix = attributes.get(PerFieldDocValuesFormat.PER_FIELD_SUFFIX_KEY); if (dvSuffix != null) { dvSuffixes.put(dvSuffix, field.name); } } Bits docsWithField = reader.getDocsWithField(field.name); if (docsWithField != null) { int count = 0; for (int docID = 0; docID < reader.maxDoc(); docID++) { if (docsWithField.get(docID)) { count++; } } fieldStats.docCountWithField = count; } } Directory directory = reader.directory(); for (String file : directory.listAll()) { String suffix = parseSuffix(file); long bytes = directory.fileLength(file); if (suffix != null) { switch (IndexFileNames.getExtension(file)) { case "dvd": case "dvm": stats.get(dvSuffixes.get(suffix)).dvBytes += bytes; break; case "tim": case "tip": stats.get(postingsSuffixes.get(suffix)).termsBytes += bytes; break; case "doc": stats.get(postingsSuffixes.get(suffix)).postingsBytes += bytes; break; case "pos": case "pay": stats.get(postingsSuffixes.get(suffix)).proxBytes += bytes; break; default: throw new AssertionError("unexpected suffixed file: " + file); } } else { // not a per-field file, but we can hackishly do this for the points case. if ("dii".equals(IndexFileNames.getExtension(file))) { System.err.println( "retrieving per-field point usage, if you see a scary corruption error, its probably just this tool!!!!"); try (ChecksumIndexInput in = directory.openChecksumInput(file, IOContext.READONCE)) { // fail hard if its not exactly the version we do this hack for. CodecUtil.checkIndexHeader(in, "Lucene60PointsFormatMeta", 0, 0, reader.getSegmentInfo().info.getId(), ""); int fieldCount = in.readVInt(); // strangely, bkd offsets are not in any guaranteed order TreeMap<Long, String> offsetToField = new TreeMap<>(); for (int i = 0; i < fieldCount; i++) { int field = in.readVInt(); long offset = in.readVLong(); offsetToField.put(offset, reader.getFieldInfos().fieldInfo(field).name); } // now we can traverse in order long previousOffset = 0; for (Map.Entry<Long, String> entry : offsetToField.entrySet()) { long offset = entry.getKey(); String field = entry.getValue(); stats.get(field).pointsBytes += (offset - previousOffset); previousOffset = offset; } CodecUtil.checkFooter(in); } } } } return new TreeSet<FieldStats>(stats.values()); }
From source file:perf.DiskUsage.java
License:Apache License
static Set<FieldStats> analyzeFields(SegmentReader reader) throws Exception { Map<String, FieldStats> stats = new HashMap<>(); Map<String, String> dvSuffixes = new HashMap<>(); Map<String, String> postingsSuffixes = new HashMap<>(); for (FieldInfo field : reader.getFieldInfos()) { FieldStats fieldStats = new FieldStats(field.name); stats.put(field.name, fieldStats); Map<String, String> attributes = field.attributes(); if (attributes != null) { String postingsSuffix = attributes.get(PerFieldPostingsFormat.PER_FIELD_SUFFIX_KEY); if (postingsSuffix != null) { postingsSuffixes.put(postingsSuffix, field.name); }//from ww w . jav a 2 s . co m String dvSuffix = attributes.get(PerFieldDocValuesFormat.PER_FIELD_SUFFIX_KEY); if (dvSuffix != null) { dvSuffixes.put(dvSuffix, field.name); } } DocIdSetIterator docsWithField; switch (field.getDocValuesType()) { case NUMERIC: docsWithField = reader.getNumericDocValues(field.name); break; case BINARY: docsWithField = reader.getBinaryDocValues(field.name); break; case SORTED: docsWithField = reader.getSortedDocValues(field.name); break; case SORTED_NUMERIC: docsWithField = reader.getSortedNumericDocValues(field.name); break; case SORTED_SET: docsWithField = reader.getSortedSetDocValues(field.name); break; case NONE: docsWithField = null; break; default: docsWithField = null; break; } if (docsWithField != null) { int count = 0; while (docsWithField.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { count++; } fieldStats.docCountWithField = count; } } Directory directory = reader.directory(); for (String file : directory.listAll()) { String suffix = parseSuffix(file); long bytes = directory.fileLength(file); if (suffix != null) { switch (IndexFileNames.getExtension(file)) { case "dvd": case "dvm": stats.get(dvSuffixes.get(suffix)).dvBytes += bytes; break; case "tim": case "tip": stats.get(postingsSuffixes.get(suffix)).termsBytes += bytes; break; case "doc": stats.get(postingsSuffixes.get(suffix)).postingsBytes += bytes; break; case "pos": case "pay": stats.get(postingsSuffixes.get(suffix)).proxBytes += bytes; break; default: throw new AssertionError("unexpected suffixed file: " + file); } } else { // not a per-field file, but we can hackishly do this for the points case. if ("dii".equals(IndexFileNames.getExtension(file))) { System.err.println( "retrieving per-field point usage, if you see a scary corruption error, its probably just this tool!!!!"); try (ChecksumIndexInput in = directory.openChecksumInput(file, IOContext.READONCE)) { // fail hard if its not exactly the version we do this hack for. CodecUtil.checkIndexHeader(in, "Lucene60PointsFormatMeta", 0, 0, reader.getSegmentInfo().info.getId(), ""); int fieldCount = in.readVInt(); // strangely, bkd offsets are not in any guaranteed order TreeMap<Long, String> offsetToField = new TreeMap<>(); for (int i = 0; i < fieldCount; i++) { int field = in.readVInt(); long offset = in.readVLong(); offsetToField.put(offset, reader.getFieldInfos().fieldInfo(field).name); } // now we can traverse in order long previousOffset = 0; for (Map.Entry<Long, String> entry : offsetToField.entrySet()) { long offset = entry.getKey(); String field = entry.getValue(); stats.get(field).pointsBytes += (offset - previousOffset); previousOffset = offset; } CodecUtil.checkFooter(in); } } } } return new TreeSet<FieldStats>(stats.values()); }