List of usage examples for org.apache.lucene.index StoredFieldVisitor StoredFieldVisitor
protected StoredFieldVisitor()
From source file:com.github.hotware.lucene.extension.highlight.BaseObjectFragmentsBuilder.java
License:BEER-WARE LICENSE
private static Field[] getFields(IndexReader reader, int docId, final String fieldName) throws IOException { // according to javadoc, doc.getFields(fieldName) cannot be used with // lazy loaded field??? final List<Field> fields = new ArrayList<>(); reader.document(docId, new StoredFieldVisitor() { @Override/* w w w . j av a 2 s .c om*/ public void stringField(FieldInfo fieldInfo, String value) { FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); fields.add(new Field(fieldInfo.name, value, ft)); } @Override public Status needsField(FieldInfo fieldInfo) { return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO; } }); return fields.toArray(new Field[fields.size()]); }
From source file:com.o19s.solr.swan.highlight.BaseFragmentsBuilder.java
License:Apache License
protected Field[] getFields(IndexReader reader, int docId, final String fieldName) throws IOException { // according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field??? final List<Field> fields = new ArrayList<Field>(); reader.document(docId, new StoredFieldVisitor() { @Override//from w w w . j av a2s .c o m public void stringField(FieldInfo fieldInfo, String value) { FieldType ft = new FieldType(TextField.TYPE_STORED); ft.setStoreTermVectors(fieldInfo.hasVectors()); fields.add(new Field(fieldInfo.name, value, ft)); } @Override public Status needsField(FieldInfo fieldInfo) { return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO; } }); return fields.toArray(new Field[fields.size()]); }
From source file:kaist.irproject.lucene.trec.DocNameExtractor.java
License:Apache License
/** * Extract the name of the input doc from the index. * @param searcher access to the index.//from w ww . jav a 2 s . c o m * @param docid ID of doc whose name is needed. * @return the name of the input doc as extracted from the index. * @throws IOException if cannot extract the doc name from the index. */ public String docName(IndexSearcher searcher, int docid) throws IOException { final List<String> name = new ArrayList<>(); searcher.getIndexReader().document(docid, new StoredFieldVisitor() { @Override public void stringField(FieldInfo fieldInfo, String value) { name.add(value); } @Override public Status needsField(FieldInfo fieldInfo) { if (!name.isEmpty()) { return Status.STOP; } else if (fieldInfo.name.equals(docNameField)) { return Status.YES; } else { return Status.NO; } } }); if (name.size() != 0) { return name.get(0); } else { return null; } }
From source file:lucene.security.index.SecureAtomicReader.java
License:Apache License
@Override public void document(int docID, final StoredFieldVisitor visitor) throws IOException { if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_READ, docID)) { in.document(docID, visitor);//ww w .j a va 2s . c om return; } if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_DISCOVER, docID)) { // TODO add way to perform code when visitor runs here.... in.document(docID, new StoredFieldVisitor() { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { if (_accessControl.canDiscoverField(fieldInfo.name)) { return visitor.needsField(fieldInfo); } else { return Status.NO; } } @Override public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { visitor.binaryField(fieldInfo, value); } @Override public void stringField(FieldInfo fieldInfo, String value) throws IOException { visitor.stringField(fieldInfo, value); } @Override public void intField(FieldInfo fieldInfo, int value) throws IOException { visitor.intField(fieldInfo, value); } @Override public void longField(FieldInfo fieldInfo, long value) throws IOException { visitor.longField(fieldInfo, value); } @Override public void floatField(FieldInfo fieldInfo, float value) throws IOException { visitor.floatField(fieldInfo, value); } @Override public void doubleField(FieldInfo fieldInfo, double value) throws IOException { visitor.doubleField(fieldInfo, value); } }); return; } }
From source file:org.apache.blur.manager.IndexManager.java
License:Apache License
private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException { reader.document(docId, new StoredFieldVisitor() { @Override/*from w w w .j a v a2 s . c o m*/ public Status needsField(FieldInfo fieldInfo) throws IOException { if (ROW_ID.equals(fieldInfo.name)) { return StoredFieldVisitor.Status.STOP; } return StoredFieldVisitor.Status.NO; } }); return reader.document(docId).get(ROW_ID); }
From source file:org.apache.solr.handler.component.TermVectorComponent.java
License:Apache License
@Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (!params.getBool(COMPONENT_NAME, false)) { return;/*from ww w . ja v a2s. c o m*/ } NamedList<Object> termVectors = new NamedList<Object>(); rb.rsp.add(TERM_VECTORS, termVectors); IndexSchema schema = rb.req.getSchema(); SchemaField keyField = schema.getUniqueKeyField(); String uniqFieldName = null; if (keyField != null) { uniqFieldName = keyField.getName(); termVectors.add("uniqueKeyFieldName", uniqFieldName); } FieldOptions allFields = new FieldOptions(); //figure out what options we have, and try to get the appropriate vector allFields.termFreq = params.getBool(TermVectorParams.TF, false); allFields.positions = params.getBool(TermVectorParams.POSITIONS, false); allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false); allFields.docFreq = params.getBool(TermVectorParams.DF, false); allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false); //boolean cacheIdf = params.getBool(TermVectorParams.IDF, false); //short cut to all values. if (params.getBool(TermVectorParams.ALL, false)) { allFields.termFreq = true; allFields.positions = true; allFields.offsets = true; allFields.docFreq = true; allFields.tfIdf = true; } //Build up our per field mapping Map<String, FieldOptions> fieldOptions = new HashMap<String, FieldOptions>(); NamedList<List<String>> warnings = new NamedList<List<String>>(); List<String> noTV = new ArrayList<String>(); List<String> noPos = new ArrayList<String>(); List<String> noOff = new ArrayList<String>(); Set<String> fields = getFields(rb); if (null != fields) { //we have specific fields to retrieve, or no fields for (String field : fields) { // workarround SOLR-3523 if (null == field || "score".equals(field)) continue; // we don't want to issue warnings about the uniqueKey field // since it can cause lots of confusion in distributed requests // where the uniqueKey field is injected into the fl for merging final boolean fieldIsUniqueKey = field.equals(uniqFieldName); SchemaField sf = schema.getFieldOrNull(field); if (sf != null) { if (sf.storeTermVector()) { FieldOptions option = fieldOptions.get(field); if (option == null) { option = new FieldOptions(); option.fieldName = field; fieldOptions.put(field, option); } //get the per field mappings option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq); option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq); option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf); //Validate these are even an option option.positions = params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions); if (option.positions && !sf.storeTermPositions() && !fieldIsUniqueKey) { noPos.add(field); } option.offsets = params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets); if (option.offsets && !sf.storeTermOffsets() && !fieldIsUniqueKey) { noOff.add(field); } } else {//field doesn't have term vectors if (!fieldIsUniqueKey) noTV.add(field); } } else { //field doesn't exist throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field); } } } //else, deal with all fields // NOTE: currently all typs of warnings are schema driven, and garunteed // to be consistent across all shards - if additional types of warnings // are added that might be differnet between shards, finishStage() needs // to be changed to account for that. boolean hasWarnings = false; if (!noTV.isEmpty()) { warnings.add("noTermVectors", noTV); hasWarnings = true; } if (!noPos.isEmpty()) { warnings.add("noPositions", noPos); hasWarnings = true; } if (!noOff.isEmpty()) { warnings.add("noOffsets", noOff); hasWarnings = true; } if (hasWarnings) { termVectors.add("warnings", warnings); } DocListAndSet listAndSet = rb.getResults(); List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS)); Iterator<Integer> iter; if (docIds != null && !docIds.isEmpty()) { iter = docIds.iterator(); } else { DocList list = listAndSet.docList; iter = list.iterator(); } SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader reader = searcher.getIndexReader(); //the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors //Only load the id field to get the uniqueKey of that //field final String finalUniqFieldName = uniqFieldName; final List<String> uniqValues = new ArrayList<String>(); // TODO: is this required to be single-valued? if so, we should STOP // once we find it... final StoredFieldVisitor getUniqValue = new StoredFieldVisitor() { @Override public void stringField(FieldInfo fieldInfo, String value) { uniqValues.add(value); } @Override public void intField(FieldInfo fieldInfo, int value) { uniqValues.add(Integer.toString(value)); } @Override public void longField(FieldInfo fieldInfo, long value) { uniqValues.add(Long.toString(value)); } @Override public Status needsField(FieldInfo fieldInfo) { return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO; } }; TermsEnum termsEnum = null; while (iter.hasNext()) { Integer docId = iter.next(); NamedList<Object> docNL = new NamedList<Object>(); if (keyField != null) { reader.document(docId, getUniqValue); String uniqVal = null; if (uniqValues.size() != 0) { uniqVal = uniqValues.get(0); uniqValues.clear(); docNL.add("uniqueKey", uniqVal); termVectors.add(uniqVal, docNL); } } else { // support for schemas w/o a unique key, termVectors.add("doc-" + docId, docNL); } if (null != fields) { for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) { final String field = entry.getKey(); final Terms vector = reader.getTermVector(docId, field); if (vector != null) { termsEnum = vector.iterator(termsEnum); mapOneVector(docNL, entry.getValue(), reader, docId, vector.iterator(termsEnum), field); } } } else { // extract all fields final Fields vectors = reader.getTermVectors(docId); for (String field : vectors) { Terms terms = vectors.terms(field); if (terms != null) { termsEnum = terms.iterator(termsEnum); mapOneVector(docNL, allFields, reader, docId, termsEnum, field); } } } } }
From source file:org.elasticsearch.deps.lucene.SimpleLuceneTests.java
License:Apache License
/** * Here, we verify that the order that we add fields to a document counts, and not the lexi order * of the field. This means that heavily accessed fields that use field selector should be added * first (with load and break)./*w w w .ja v a 2 s . c o m*/ */ @Test public void testOrdering() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER)); Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); document.add(new TextField("#id", "1", Field.Store.YES)); indexWriter.addDocument(document); IndexReader reader = DirectoryReader.open(indexWriter, true); IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); final ArrayList<String> fieldsOrder = new ArrayList<String>(); searcher.doc(topDocs.scoreDocs[0].doc, new StoredFieldVisitor() { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { fieldsOrder.add(fieldInfo.name); return Status.YES; } }); assertThat(fieldsOrder.size(), equalTo(2)); assertThat(fieldsOrder.get(0), equalTo("_id")); assertThat(fieldsOrder.get(1), equalTo("#id")); indexWriter.close(); }
From source file:org.elasticsearch.xpack.core.security.authz.accesscontrol.FieldSubsetReader.java
License:Open Source License
@Override public void document(final int docID, final StoredFieldVisitor visitor) throws IOException { super.document(docID, new StoredFieldVisitor() { @Override//from ww w.ja va 2 s. c o m public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { if (SourceFieldMapper.NAME.equals(fieldInfo.name)) { // for _source, parse, filter out the fields we care about, and serialize back downstream BytesReference bytes = new BytesArray(value); Tuple<XContentType, Map<String, Object>> result = XContentHelper.convertToMap(bytes, true); Map<String, Object> transformedSource = filter(result.v2(), filter, 0); XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()) .map(transformedSource); visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder))); } else { visitor.binaryField(fieldInfo, value); } } @Override public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { visitor.stringField(fieldInfo, value); } @Override public void intField(FieldInfo fieldInfo, int value) throws IOException { visitor.intField(fieldInfo, value); } @Override public void longField(FieldInfo fieldInfo, long value) throws IOException { visitor.longField(fieldInfo, value); } @Override public void floatField(FieldInfo fieldInfo, float value) throws IOException { visitor.floatField(fieldInfo, value); } @Override public void doubleField(FieldInfo fieldInfo, double value) throws IOException { visitor.doubleField(fieldInfo, value); } @Override public Status needsField(FieldInfo fieldInfo) throws IOException { return hasField(fieldInfo.name) ? visitor.needsField(fieldInfo) : Status.NO; } }); }
From source file:tech.beshu.ror.es.security.DocumentFieldReader.java
License:Open Source License
@Override public void document(int docID, StoredFieldVisitor visitor) throws IOException { super.document(docID, new StoredFieldVisitor() { @Override//from w ww. j av a 2 s . c o m public Status needsField(FieldInfo fieldInfo) throws IOException { return policy.canKeep(fieldInfo.name) ? visitor.needsField(fieldInfo) : Status.NO; } @Override public int hashCode() { return visitor.hashCode(); } @Override public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { visitor.stringField(fieldInfo, value); } @Override public boolean equals(Object obj) { return visitor.equals(obj); } @Override public void doubleField(FieldInfo fieldInfo, double value) throws IOException { visitor.doubleField(fieldInfo, value); } @Override public void floatField(FieldInfo fieldInfo, float value) throws IOException { visitor.floatField(fieldInfo, value); } @Override public void intField(FieldInfo fieldInfo, int value) throws IOException { visitor.intField(fieldInfo, value); } @Override public void longField(FieldInfo fieldInfo, long value) throws IOException { visitor.longField(fieldInfo, value); } @Override public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { if (!"_source".equals(fieldInfo.name)) { visitor.binaryField(fieldInfo, value); return; } Tuple<XContentType, Map<String, Object>> xContentTypeMapTuple = XContentHelper .convertToMap(new BytesArray(value), false, XContentType.JSON); Map<String, Object> map = xContentTypeMapTuple.v2(); Iterator<String> it = map.keySet().iterator(); while (it.hasNext()) { if (!policy.canKeep(it.next())) { it.remove(); } } final XContentBuilder xBuilder = XContentBuilder.builder(xContentTypeMapTuple.v1().xContent()) .map(map); ByteArrayOutputStream out = new ByteArrayOutputStream(); BytesReference.bytes(xBuilder).writeTo(out); visitor.binaryField(fieldInfo, out.toByteArray()); } }); }