Example usage for org.apache.lucene.search IndexSearcher getIndexReader

List of usage examples for org.apache.lucene.search IndexSearcher getIndexReader

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.search.*
  5. IndexSearcher
  6. getIndexReader

Introduction

In this page you can find the example usage for org.apache.lucene.search IndexSearcher getIndexReader.

Prototype

public IndexReader getIndexReader()

Source Link

Document

Return the IndexReader this searches.

Usage

From source file:com.tuplejump.stargate.PerVNodeIndexContainer.java

License:Apache License

@Override
public <T> T search(SearcherCallback<T> searcherCallback) {
    List<IndexReader> indexReaders = new ArrayList<>();
    Map<Indexer, IndexSearcher> indexSearchers = new HashMap<>();
    for (Map.Entry<Range<Token>, Indexer> entry : indexers.entrySet()) {
        Range<Token> range = entry.getKey();
        boolean intersects = intersects(searcherCallback.filterRange(), searcherCallback.isSingleToken(),
                searcherCallback.isFullRange(), range);
        if (intersects) {
            Indexer indexer = entry.getValue();
            IndexSearcher searcher = indexer.acquire();
            indexSearchers.put(indexer, searcher);
            indexReaders.add(searcher.getIndexReader());
        }/*from ww w  .java2s .co  m*/
    }
    IndexReader[] indexReadersArr = new IndexReader[indexReaders.size()];
    indexReaders.toArray(indexReadersArr);
    MultiReader multiReader = null;
    try {
        multiReader = new MultiReader(indexReadersArr, false);
        IndexSearcher allSearcher = new IndexSearcher(multiReader, executorService);
        return searcherCallback.doWithSearcher(allSearcher);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        try {
            if (multiReader != null)
                multiReader.close();
        } catch (IOException e) {
            logger.error("Could not close reader", e);
        }
        for (Map.Entry<Indexer, IndexSearcher> entry : indexSearchers.entrySet()) {
            entry.getKey().release(entry.getValue());
        }
    }
}

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void queryIndexSingle(String selfLink, EnumSet<QueryOption> options, Operation op) throws Throwable {
    IndexWriter w = this.writer;
    if (w == null) {
        op.fail(new CancellationException());
        return;/*from w  w  w  .  j a v a 2  s. co m*/
    }

    IndexSearcher s = updateSearcher(selfLink, 1, w);
    long start = Utils.getNowMicrosUtc();
    TopDocs hits = searchLatestVersion(selfLink, s);
    long end = Utils.getNowMicrosUtc();
    if (hits.totalHits == 0) {
        op.complete();
        return;
    }

    if (hasOption(ServiceOption.INSTRUMENTATION)) {
        ServiceStat st = getHistogramStat(STAT_NAME_QUERY_SINGLE_DURATION_MICROS);
        setStat(st, end - start);
    }

    Document doc = s.getIndexReader().document(hits.scoreDocs[0].doc, this.fieldsToLoadWithExpand);

    if (checkAndDeleteExpiratedDocuments(selfLink, s, hits.scoreDocs[0].doc, doc, Utils.getNowMicrosUtc())) {
        op.complete();
        return;
    }

    BytesRef binaryState = doc.getBinaryValue(LUCENE_FIELD_NAME_BINARY_SERIALIZED_STATE);

    if (binaryState != null) {
        ServiceDocument state = (ServiceDocument) Utils.fromDocumentBytes(binaryState.bytes, binaryState.offset,
                binaryState.length);
        op.setBodyNoCloning(state);
    }
    op.complete();
}

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void processQueryResults(ServiceOption targetIndex, EnumSet<QueryOption> options, IndexSearcher s,
        ServiceDocumentQueryResult rsp, ScoreDoc[] hits, long queryStartTimeMicros) throws Throwable {

    Set<String> fieldsToLoad = this.fieldsToLoadNoExpand;
    if (options.contains(QueryOption.EXPAND_CONTENT)) {
        fieldsToLoad = this.fieldsToLoadWithExpand;
    }/* w  w w . ja v a 2  s  .c o m*/

    // Keep duplicates out
    Set<String> uniques = new LinkedHashSet<>();
    final boolean hasCountOption = options.contains(QueryOption.COUNT);

    Map<String, Long> latestVersions = new HashMap<>();
    for (ScoreDoc sd : hits) {
        Document d = s.getIndexReader().document(sd.doc, fieldsToLoad);
        String link = d.get(ServiceDocument.FIELD_NAME_SELF_LINK).intern();
        IndexableField versionField = d.getField(ServiceDocument.FIELD_NAME_VERSION);
        Long documentVersion = versionField.numericValue().longValue();

        // We first determine what is the latest document version.
        // We then use the latest version to determine if the current document result is relevant.
        Long latestVersion = latestVersions.get(link);
        if (latestVersion == null) {
            latestVersion = getLatestVersion(s, link);
            latestVersions.put(link, latestVersion);
        }

        boolean isDeleted = DELETE_ACTION.equals(d.get(ServiceDocument.FIELD_NAME_UPDATE_ACTION));

        if (isDeleted && !options.contains(QueryOption.INCLUDE_DELETED)) {
            // ignore a document if its marked deleted and it has the latest version
            if (documentVersion >= latestVersion) {
                uniques.remove(link);
            }
            continue;
        }

        if (!options.contains(QueryOption.INCLUDE_ALL_VERSIONS)) {
            if (documentVersion < latestVersion) {
                continue;
            }
        } else {
            // decorate link with version
            link = UriUtils.buildPathWithVersion(link, documentVersion);
        }

        if (checkAndDeleteExpiratedDocuments(link, s, sd.doc, d, queryStartTimeMicros)) {
            // ignore all document versions if the link has expired
            latestVersions.put(link, Long.MAX_VALUE);
            continue;
        }

        if (hasCountOption) {
            // count unique instances of this link
            uniques.add(link);
            // we only want to count the link once, so set version to highest
            latestVersions.put(link, Long.MAX_VALUE);
            continue;
        }

        if (options.contains(QueryOption.EXPAND_CONTENT)) {
            String json = null;
            ServiceDocument state = getStateFromLuceneDocument(d, link);
            if (state == null) {
                // support reading JSON serialized state for backwards compatibility
                json = d.get(LUCENE_FIELD_NAME_JSON_SERIALIZED_STATE);
                if (json == null) {
                    continue;
                }
            } else {
                json = Utils.toJson(state);
            }
            if (!rsp.documents.containsKey(link)) {
                rsp.documents.put(link, new JsonParser().parse(json).getAsJsonObject());
            }
        }
        uniques.add(link);
    }

    if (hasCountOption) {
        rsp.documentCount = Long.valueOf(uniques.size());
    } else {
        rsp.documentLinks.addAll(uniques);
        rsp.documentCount = Long.valueOf(rsp.documentLinks.size());
    }
}

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private long getLatestVersion(IndexSearcher s, String link) throws IOException {
    IndexableField versionField;//  w  w  w.j a v  a2s .co  m
    long latestVersion;
    TopDocs td = searchLatestVersion(link, s);
    Document latestVersionDoc = s.getIndexReader().document(td.scoreDocs[0].doc, this.fieldsToLoadNoExpand);
    versionField = latestVersionDoc.getField(ServiceDocument.FIELD_NAME_VERSION);
    latestVersion = versionField.numericValue().longValue();
    return latestVersion;
}

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private boolean checkAndDeleteExpiratedDocuments(String link, IndexSearcher searcher, Integer docId,
        Document doc, long now) throws Throwable {
    long expiration = 0;
    boolean hasExpired = false;
    IndexableField expirationValue = doc.getField(ServiceDocument.FIELD_NAME_EXPIRATION_TIME_MICROS);
    if (expirationValue != null) {
        expiration = expirationValue.numericValue().longValue();
        hasExpired = expiration <= now;
    }//  ww w . j  av a  2  s .  c  o m

    if (hasExpired) {
        adjustStat(STAT_NAME_DOCUMENT_EXPIRATION_COUNT, 1);
        if (searcher != null) {
            doc = searcher.getIndexReader().document(docId, this.fieldsToLoadWithExpand);
        }
        ServiceDocument s = getStateFromLuceneDocument(doc, link);
        deleteAllDocumentsForSelfLink(Operation.createDelete(null), s);
        return true;
    }

    return false;
}

From source file:com.vmware.dcp.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void applyDocumentExpirationPolicy(IndexWriter w) throws Throwable {
    IndexSearcher s = updateSearcher(null, Integer.MAX_VALUE, w);
    if (s == null) {
        return;//  w w w . j a v a2s  . com
    }

    long expirationUpperBound = Utils.getNowMicrosUtc();

    NumericRangeQuery<Long> versionQuery = NumericRangeQuery.newLongRange(
            ServiceDocument.FIELD_NAME_EXPIRATION_TIME_MICROS, 1L, expirationUpperBound, true, true);

    TopDocs results = s.search(versionQuery, Integer.MAX_VALUE);
    if (results.totalHits == 0) {
        return;
    }

    // The expiration query will return all versions for a link. Use a set so we only delete once per link
    Set<String> links = new HashSet<>();
    long now = Utils.getNowMicrosUtc();
    for (ScoreDoc sd : results.scoreDocs) {
        Document d = s.getIndexReader().document(sd.doc, this.fieldsToLoadNoExpand);
        String link = d.get(ServiceDocument.FIELD_NAME_SELF_LINK).intern();
        IndexableField versionField = d.getField(ServiceDocument.FIELD_NAME_VERSION);
        long versionExpired = versionField.numericValue().longValue();
        long latestVersion = this.getLatestVersion(s, link);
        if (versionExpired < latestVersion) {
            continue;
        }
        if (!links.add(link)) {
            continue;
        }
        checkAndDeleteExpiratedDocuments(link, s, sd.doc, d, now);
    }
}

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java

License:Open Source License

private void queryIndexSingle(String selfLink, Operation op, Long version) throws Throwable {
    IndexWriter w = this.writer;
    if (w == null) {
        op.fail(new CancellationException());
        return;/*from www  .  j a  v  a  2s  .c o m*/
    }

    IndexSearcher s = updateSearcher(selfLink, 1, w);
    long start = 0;
    long end = 0;
    if (hasOption(ServiceOption.INSTRUMENTATION)) {
        start = System.nanoTime();
    }
    TopDocs hits = searchByVersion(selfLink, s, version);
    if (hasOption(ServiceOption.INSTRUMENTATION)) {
        end = System.nanoTime();
    }

    if (hasOption(ServiceOption.INSTRUMENTATION)) {
        ServiceStat st = getHistogramStat(STAT_NAME_QUERY_SINGLE_DURATION_MICROS);
        setStat(st, (end - start) / 1000.0);
    }

    if (hits.totalHits == 0) {
        op.complete();
        return;
    }

    Document doc = s.getIndexReader().document(hits.scoreDocs[0].doc, this.fieldsToLoadWithExpand);

    if (checkAndDeleteExpiratedDocuments(selfLink, s, hits.scoreDocs[0].doc, doc, Utils.getNowMicrosUtc())) {
        op.complete();
        return;
    }

    ServiceDocument sd = getStateFromLuceneDocument(doc, selfLink);
    op.setBodyNoCloning(sd).complete();
}

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java

License:Open Source License

private ScoreDoc processQueryResults(QuerySpecification qs, EnumSet<QueryOption> options, int resultLimit,
        IndexSearcher s, ServiceDocumentQueryResult rsp, ScoreDoc[] hits, long queryStartTimeMicros)
        throws Throwable {

    ScoreDoc lastDocVisited = null;/*  w  ww.j  a v a 2 s. com*/
    Set<String> fieldsToLoad = this.fieldsToLoadNoExpand;
    if (options.contains(QueryOption.EXPAND_CONTENT) || options.contains(QueryOption.OWNER_SELECTION)) {
        fieldsToLoad = this.fieldsToLoadWithExpand;
    }

    if (options.contains(QueryOption.SELECT_LINKS)) {
        fieldsToLoad = new HashSet<>(fieldsToLoad);
        for (QueryTask.QueryTerm link : qs.linkTerms) {
            fieldsToLoad.add(link.propertyName);
        }
    }

    // Keep duplicates out
    Set<String> uniques = new LinkedHashSet<>(rsp.documentLinks);
    final boolean hasCountOption = options.contains(QueryOption.COUNT);
    final boolean hasIncludeAllVersionsOption = options.contains(QueryOption.INCLUDE_ALL_VERSIONS);
    Set<String> linkWhiteList = null;
    if (qs != null && qs.context != null && qs.context.documentLinkWhiteList != null) {
        linkWhiteList = qs.context.documentLinkWhiteList;
    }

    Map<String, Long> latestVersions = new HashMap<>();
    for (ScoreDoc sd : hits) {
        if (uniques.size() >= resultLimit) {
            break;
        }

        lastDocVisited = sd;
        Document d = s.getIndexReader().document(sd.doc, fieldsToLoad);
        String link = d.get(ServiceDocument.FIELD_NAME_SELF_LINK);

        // ignore results not in supplied white list
        if (linkWhiteList != null && !linkWhiteList.contains(link)) {
            continue;
        }

        IndexableField versionField = d.getField(ServiceDocument.FIELD_NAME_VERSION);
        Long documentVersion = versionField.numericValue().longValue();

        Long latestVersion = null;

        if (hasIncludeAllVersionsOption) {
            // Decorate link with version. If a document is marked deleted, at any version,
            // we will include it in the results
            link = UriUtils.buildPathWithVersion(link, documentVersion);
        } else {
            // We first determine what is the latest document version.
            // We then use the latest version to determine if the current document result is relevant.
            latestVersion = latestVersions.get(link);
            if (latestVersion == null) {
                latestVersion = getLatestVersion(s, link);
                latestVersions.put(link, latestVersion);
            }

            if (documentVersion < latestVersion) {
                continue;
            }

            boolean isDeleted = Action.DELETE.toString()
                    .equals(d.get(ServiceDocument.FIELD_NAME_UPDATE_ACTION));

            if (isDeleted && !options.contains(QueryOption.INCLUDE_DELETED)) {
                // ignore a document if its marked deleted and it has the latest version
                if (documentVersion >= latestVersion) {
                    uniques.remove(link);
                    if (rsp.documents != null) {
                        rsp.documents.remove(link);
                    }
                    if (rsp.selectedLinksPerDocument != null) {
                        rsp.selectedLinksPerDocument.remove(link);
                    }
                }
                continue;
            }
        }

        if (checkAndDeleteExpiratedDocuments(link, s, sd.doc, d, queryStartTimeMicros)) {
            // ignore all document versions if the link has expired
            latestVersions.put(link, Long.MAX_VALUE);
            continue;
        }

        if (hasCountOption) {
            // count unique instances of this link
            uniques.add(link);
            // we only want to count the link once, so set version to highest
            latestVersions.put(link, Long.MAX_VALUE);
            continue;
        }

        String json = null;
        ServiceDocument state = null;

        if (options.contains(QueryOption.EXPAND_CONTENT) || options.contains(QueryOption.OWNER_SELECTION)) {
            state = getStateFromLuceneDocument(d, link);
            if (state == null) {
                // support reading JSON serialized state for backwards compatibility
                json = d.get(LUCENE_FIELD_NAME_JSON_SERIALIZED_STATE);
                if (json == null) {
                    continue;
                }
            } else {
                json = Utils.toJson(state);
            }
        }

        if (options.contains(QueryOption.OWNER_SELECTION)) {
            if (!processQueryResultsForOwnerSelection(json, state)) {
                continue;
            }
        }

        if (options.contains(QueryOption.EXPAND_CONTENT) && !rsp.documents.containsKey(link)) {
            if (options.contains(QueryOption.EXPAND_BUILTIN_CONTENT_ONLY)) {
                ServiceDocument stateClone = new ServiceDocument();
                state.copyTo(stateClone);
                rsp.documents.put(link, stateClone);
            } else {
                JsonObject jo = new JsonParser().parse(json).getAsJsonObject();
                rsp.documents.put(link, jo);
            }
        }

        if (options.contains(QueryOption.SELECT_LINKS)) {
            state = processQueryResultsForSelectLinks(s, qs, rsp, d, sd.doc, link, state);
        }

        uniques.add(link);
    }

    if (hasCountOption) {
        rsp.documentCount = (long) uniques.size();
    } else {
        rsp.documentLinks.clear();
        rsp.documentLinks.addAll(uniques);
        rsp.documentCount = (long) rsp.documentLinks.size();
    }

    return lastDocVisited;
}

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java

License:Open Source License

private ServiceDocument processQueryResultsForSelectLinks(IndexSearcher s, QuerySpecification qs,
        ServiceDocumentQueryResult rsp, Document d, int docId, String link, ServiceDocument state)
        throws Throwable {
    if (rsp.selectedLinksPerDocument == null) {
        rsp.selectedLinksPerDocument = new HashMap<>();
        rsp.selectedLinks = new HashSet<>();
    }/*w ww . j  a  v  a2  s .c o  m*/
    Map<String, String> linksPerDocument = rsp.selectedLinksPerDocument.get(link);
    if (linksPerDocument == null) {
        linksPerDocument = new HashMap<>();
        rsp.selectedLinksPerDocument.put(link, linksPerDocument);
    }

    for (QueryTask.QueryTerm qt : qs.linkTerms) {
        String linkValue = d.get(qt.propertyName);
        if (linkValue != null) {
            linksPerDocument.put(qt.propertyName, linkValue);
            rsp.selectedLinks.add(linkValue);
            continue;
        }

        // if there is no stored field with the link term property name, it might be
        // a field with a collection of links. We do not store those in lucene, they are
        // part of the binary serialized state.
        if (state == null) {
            d = s.getIndexReader().document(docId, this.fieldsToLoadWithExpand);
            state = getStateFromLuceneDocument(d, link);
            if (state == null) {
                logWarning("Skipping link %s, can not find serialized state", link);
                continue;
            }
        }

        java.lang.reflect.Field linkCollectionField = ReflectionUtils.getField(state.getClass(),
                qt.propertyName);
        if (linkCollectionField == null) {
            logWarning("Skipping link %s, can not find field", link);
            continue;
        }
        Object fieldValue = linkCollectionField.get(state);
        if (!(fieldValue instanceof Collection<?>)) {
            logWarning("Skipping link %s, field is not a collection", link);
            continue;
        }
        @SuppressWarnings("unchecked")
        Collection<String> linkCollection = (Collection<String>) fieldValue;
        int index = 0;
        for (String item : linkCollection) {
            linksPerDocument.put(QuerySpecification.buildLinkCollectionItemName(qt.propertyName, index++),
                    item);
            rsp.selectedLinks.add(item);
        }
    }
    return state;
}

From source file:com.vmware.xenon.services.common.LuceneDocumentIndexService.java

License:Open Source License

private long getLatestVersion(IndexSearcher s, String link) throws IOException {
    IndexableField versionField;/*from   w ww.  j  a v a2s. com*/
    long latestVersion;
    TopDocs td = searchByVersion(link, s, null);
    Document latestVersionDoc = s.getIndexReader().document(td.scoreDocs[0].doc, this.fieldsToLoadNoExpand);
    versionField = latestVersionDoc.getField(ServiceDocument.FIELD_NAME_VERSION);
    latestVersion = versionField.numericValue().longValue();
    return latestVersion;
}