List of usage examples for org.apache.lucene.analysis Analyzer PER_FIELD_REUSE_STRATEGY
ReuseStrategy PER_FIELD_REUSE_STRATEGY
To view the source code for org.apache.lucene.analysis Analyzer PER_FIELD_REUSE_STRATEGY.
Click Source Link
From source file:com.rocana.lucene.codec.v1.TestBlockPostingsFormat3.java
License:Apache License
public void test() throws Exception { Directory dir = newDirectory();/*from w w w . j a va 2s .co m*/ Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new MockTokenizer(); if (fieldName.contains("payloadsFixed")) { TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1); return new TokenStreamComponents(tokenizer, filter); } else if (fieldName.contains("payloadsVariable")) { TokenFilter filter = new MockVariableLengthPayloadFilter(new Random(0), tokenizer); return new TokenStreamComponents(tokenizer, filter); } else { return new TokenStreamComponents(tokenizer); } } }; IndexWriterConfig iwc = newIndexWriterConfig(analyzer); iwc.setCodec(TestUtil.alwaysPostingsFormat(new RocanaLucene50PostingsFormat())); // TODO we could actually add more fields implemented with different PFs // or, just put this test into the usual rotation? RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED); // turn this on for a cross-check docsOnlyType.setStoreTermVectors(true); docsOnlyType.setIndexOptions(IndexOptions.DOCS); FieldType docsAndFreqsType = new FieldType(TextField.TYPE_NOT_STORED); // turn this on for a cross-check docsAndFreqsType.setStoreTermVectors(true); docsAndFreqsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); FieldType positionsType = new FieldType(TextField.TYPE_NOT_STORED); // turn these on for a cross-check positionsType.setStoreTermVectors(true); positionsType.setStoreTermVectorPositions(true); positionsType.setStoreTermVectorOffsets(true); positionsType.setStoreTermVectorPayloads(true); FieldType offsetsType = new FieldType(positionsType); offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); Field field1 = new Field("field1docs", "", docsOnlyType); Field field2 = new Field("field2freqs", "", docsAndFreqsType); Field field3 = new Field("field3positions", "", positionsType); Field field4 = new Field("field4offsets", "", offsetsType); Field field5 = new Field("field5payloadsFixed", "", positionsType); Field field6 = new Field("field6payloadsVariable", "", positionsType); Field field7 = new Field("field7payloadsFixedOffsets", "", offsetsType); Field field8 = new Field("field8payloadsVariableOffsets", "", offsetsType); doc.add(field1); doc.add(field2); doc.add(field3); doc.add(field4); doc.add(field5); doc.add(field6); doc.add(field7); doc.add(field8); for (int i = 0; i < MAXDOC; i++) { String stringValue = Integer.toString(i) + " verycommon " + English.intToEnglish(i).replace('-', ' ') + " " + TestUtil.randomSimpleString(random()); field1.setStringValue(stringValue); field2.setStringValue(stringValue); field3.setStringValue(stringValue); field4.setStringValue(stringValue); field5.setStringValue(stringValue); field6.setStringValue(stringValue); field7.setStringValue(stringValue); field8.setStringValue(stringValue); iw.addDocument(doc); } iw.close(); verify(dir); TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge iwc = newIndexWriterConfig(analyzer); iwc.setCodec(TestUtil.alwaysPostingsFormat(new RocanaLucene50PostingsFormat())); iwc.setOpenMode(OpenMode.APPEND); IndexWriter iw2 = new IndexWriter(dir, iwc); iw2.forceMerge(1); iw2.close(); verify(dir); dir.close(); }
From source file:lux.index.MultiFieldAnalyzer.java
License:Mozilla Public License
public MultiFieldAnalyzer() { super(Analyzer.PER_FIELD_REUSE_STRATEGY); analyzers = new HashMap<String, Analyzer>(); analyzers.put(null, new KeywordAnalyzer()); }
From source file:org.alfresco.repo.search.impl.lucene.analysis.MLAnalayser.java
License:Open Source License
public MLAnalayser(MLAnalysisMode mlAnalaysisMode, IndexSchema schema, Mode mode) { super(Analyzer.PER_FIELD_REUSE_STRATEGY); this.mlAnalaysisMode = mlAnalaysisMode; this.schema = schema; this.mode = mode; }
From source file:org.alfresco.repo.search.impl.lucene.analysis.MLAnalayser.java
License:Open Source License
public MLAnalayser(MLAnalysisMode mlAnalaysisMode) { super(Analyzer.PER_FIELD_REUSE_STRATEGY); this.mlAnalaysisMode = mlAnalaysisMode; }
From source file:org.alfresco.solr.AlfrescoAnalyzerWrapper.java
License:Open Source License
/** * @param schema/*from w w w .j a v a 2s .c o m*/ * @param index */ public AlfrescoAnalyzerWrapper(IndexSchema schema, Mode mode) { super(Analyzer.PER_FIELD_REUSE_STRATEGY); this.schema = schema; this.mode = mode; }
From source file:org.codelibs.elasticsearch.index.analysis.FieldNameAnalyzer.java
License:Apache License
public FieldNameAnalyzer(Map<String, Analyzer> analyzers) { super(Analyzer.PER_FIELD_REUSE_STRATEGY); this.analyzers = CopyOnWriteHashMap.copyOf(analyzers); }
From source file:org.elasticsearch.index.mapper.MapperAnalyzer.java
License:Apache License
public MapperAnalyzer(MapperService mapperService) { super(Analyzer.PER_FIELD_REUSE_STRATEGY); this.mapperService = mapperService; }
From source file:org.elasticsearch.index.query.PercolateQueryBuilder.java
License:Apache License
@Override protected Query doToQuery(QueryShardContext context) throws IOException { if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) { throw new IllegalStateException("query builder must be rewritten first"); }//from www .ja va2s .co m if (document == null) { throw new IllegalStateException("nothing to percolator"); } MapperService mapperService = context.getMapperService(); DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType); DocumentMapper docMapper = docMapperForType.getDocumentMapper(); ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document)); FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer(); // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when // 'index.percolator.map_unmapped_fields_as_string' is enabled: Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override protected Analyzer getWrappedAnalyzer(String fieldName) { Analyzer analyzer = fieldNameAnalyzer.analyzers().get(fieldName); if (analyzer != null) { return analyzer; } else { return context.getAnalysisService().defaultIndexAnalyzer(); } } }; final IndexSearcher docSearcher; if (doc.docs().size() > 1) { assert docMapper.hasNestedObjects(); docSearcher = createMultiDocumentSearcher(analyzer, doc); } else { MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc.rootDoc(), analyzer, true, false); docSearcher = memoryIndex.createSearcher(); docSearcher.setQueryCache(null); } PercolatorQueryCache registry = context.getPercolatorQueryCache(); if (registry == null) { throw new QueryShardException(context, "no percolator query registry"); } PercolateQuery.Builder builder = new PercolateQuery.Builder(documentType, registry, document, docSearcher); Settings indexSettings = registry.getIndexSettings().getSettings(); if (indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) .onOrAfter(Version.V_5_0_0_alpha1)) { MappedFieldType fieldType = context.fieldMapper(field); if (fieldType == null) { throw new QueryShardException(context, "field [" + field + "] does not exist"); } if (!(fieldType instanceof PercolatorFieldMapper.PercolatorFieldType)) { throw new QueryShardException(context, "expected field [" + field + "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]"); } PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType; builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getUnknownQueryFieldName()); } else { Query percolateTypeQuery = new TermQuery( new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME)); builder.setPercolateTypeQuery(percolateTypeQuery); } return builder.build(); }
From source file:org.elasticsearch.percolator.PercolateQueryBuilder.java
License:Apache License
@Override protected Query doToQuery(QueryShardContext context) throws IOException { if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) { throw new IllegalStateException("query builder must be rewritten first"); }//from w ww . j a va 2s.c o m if (document == null) { throw new IllegalStateException("no document to percolate"); } MapperService mapperService = context.getMapperService(); DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType); DocumentMapper docMapper = docMapperForType.getDocumentMapper(); ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document)); FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer(); // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when // 'index.percolator.map_unmapped_fields_as_string' is enabled: Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) { @Override protected Analyzer getWrappedAnalyzer(String fieldName) { Analyzer analyzer = fieldNameAnalyzer.analyzers().get(fieldName); if (analyzer != null) { return analyzer; } else { return context.getAnalysisService().defaultIndexAnalyzer(); } } }; final IndexSearcher docSearcher; if (doc.docs().size() > 1) { assert docMapper.hasNestedObjects(); docSearcher = createMultiDocumentSearcher(analyzer, doc); } else { MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc.rootDoc(), analyzer, true, false); docSearcher = memoryIndex.createSearcher(); docSearcher.setQueryCache(null); } Version indexVersionCreated = context.getIndexSettings().getIndexVersionCreated(); boolean mapUnmappedFieldsAsString = context.getIndexSettings() .getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING); if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) { MappedFieldType fieldType = context.fieldMapper(field); if (fieldType == null) { throw new QueryShardException(context, "field [" + field + "] does not exist"); } if (!(fieldType instanceof PercolatorFieldMapper.FieldType)) { throw new QueryShardException(context, "expected field [" + field + "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]"); } PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType; PercolateQuery.QueryStore queryStore = createStore(pft, context, mapUnmappedFieldsAsString); return pft.percolateQuery(documentType, queryStore, document, docSearcher); } else { Query percolateTypeQuery = new TermQuery( new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME)); PercolateQuery.QueryStore queryStore = createLegacyStore(context, mapUnmappedFieldsAsString); return new PercolateQuery(documentType, queryStore, document, percolateTypeQuery, docSearcher, new MatchNoDocsQuery("pre 5.0.0-alpha1 index, no verified matches")); } }
From source file:org.ihtsdo.otf.query.lucene.PerFieldAnalyzer.java
License:Apache License
public PerFieldAnalyzer() { super(Analyzer.PER_FIELD_REUSE_STRATEGY); }