Example usage for org.apache.lucene.util BytesRef BytesRef

List of usage examples for org.apache.lucene.util BytesRef BytesRef

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.util.*
  5. BytesRef
  6. BytesRef

Introduction

In this page you can find the example usage for org.apache.lucene.util BytesRef BytesRef.

Prototype

public BytesRef(CharSequence text)

Source Link

Document

Initialize the byte[] from the UTF8 bytes for the provided String.

Usage

From source file:com.xiaomi.linden.core.search.query.QueryConstructor.java

License:Apache License

public static BytesRef bytesRefVal(String val) {
    return val == null ? null : new BytesRef(val);
}

From source file:CopulaResources.TermCooccurence.java

public static TermCooccurence generateCooccurencebyClass(IndexReader indexReader, String classFieldName,
        String textFieldName, Analyzer analyzer, int minFreq, int maxFreq) throws IOException {
    System.out.println(":::Generating Term-Pair List:::");
    TermCooccurence CooccurList = new TermCooccurence();
    Terms classes = MultiFields.getTerms(indexReader, classFieldName);
    if (classes != null) {
        TermsEnum classesEnum = classes.iterator();
        BytesRef nextClass;/* w  w  w  . j a  v  a 2s . c o  m*/
        while ((nextClass = classesEnum.next()) != null) {
            if (nextClass.length > 0) {

                Term term = new Term(classFieldName, nextClass);
                String tpClass = nextClass.utf8ToString();
                BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
                booleanQuery.add(new BooleanClause(new TermQuery(term), BooleanClause.Occur.MUST));
                IndexSearcher indexSearcher = new IndexSearcher(indexReader);
                TopDocs topDocs;
                topDocs = indexSearcher.search(booleanQuery.build(), indexReader.numDocs());

                for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
                    IndexableField[] storableFields = indexSearcher.doc(scoreDoc.doc).getFields(textFieldName);
                    for (IndexableField singleStorableField : storableFields) {
                        if (singleStorableField != null) {
                            BytesRef text = new BytesRef(singleStorableField.stringValue());
                            generateCooccurences(text.utf8ToString(), analyzer, CooccurList, tpClass);
                        }
                    }
                }
                CooccurList.trimbyFreq(tpClass, minFreq, maxFreq);

            }
        }
    }
    System.out.println(":::Generation Complete:::");
    return CooccurList;
}

From source file:CopulaResources.TermCooccurence.java

public static void generateCooccurencebyClass(IndexReader indexReader, String classFieldName,
        String textFieldName, Analyzer analyzer, int minFreq, int maxFreq, Path saveDir) throws IOException {
    System.out.println(":::Generating Term-Pair List:::");
    TermCooccurence CooccurList = new TermCooccurence();
    Terms classes = MultiFields.getTerms(indexReader, classFieldName);
    if (classes != null) {
        TermsEnum classesEnum = classes.iterator();
        BytesRef nextClass;/*from  w  ww . j av  a2 s. c  o  m*/
        while ((nextClass = classesEnum.next()) != null) {
            if (nextClass.length > 0) {

                Term term = new Term(classFieldName, nextClass);
                String tpClass = nextClass.utf8ToString();
                BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
                booleanQuery.add(new BooleanClause(new TermQuery(term), BooleanClause.Occur.MUST));
                IndexSearcher indexSearcher = new IndexSearcher(indexReader);
                TopDocs topDocs;
                topDocs = indexSearcher.search(booleanQuery.build(), indexReader.numDocs());

                for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
                    IndexableField[] storableFields = indexSearcher.doc(scoreDoc.doc).getFields(textFieldName);
                    for (IndexableField singleStorableField : storableFields) {
                        if (singleStorableField != null) {
                            BytesRef text = new BytesRef(singleStorableField.stringValue());
                            generateCooccurences(text.utf8ToString(), analyzer, CooccurList, tpClass);
                        }
                    }
                }
                CooccurList.trimbyFreq(tpClass, minFreq, maxFreq);

            }
        }
    }
    CooccurList.savetoFile(saveDir);
    System.out.println(":::Generation Complete:::");
}

From source file:CopulaResources.TermCooccurence.java

public static void generateNCooccurencebyClass(IndexReader indexReader, String classFieldName,
        String textFieldName, Analyzer analyzer, String direction, double percent, Path saveDir)
        throws IOException {
    System.out.println(":::Generating Term-Pair List:::");
    TermCooccurence CooccurList = new TermCooccurence();
    Terms classes = MultiFields.getTerms(indexReader, classFieldName);
    if (classes != null) {
        TermsEnum classesEnum = classes.iterator();
        BytesRef nextClass;/*from  w ww. ja v  a2s  .  com*/
        while ((nextClass = classesEnum.next()) != null) {
            if (nextClass.length > 0) {

                Term term = new Term(classFieldName, nextClass);
                String tpClass = nextClass.utf8ToString();
                BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
                booleanQuery.add(new BooleanClause(new TermQuery(term), BooleanClause.Occur.MUST));
                IndexSearcher indexSearcher = new IndexSearcher(indexReader);
                TopDocs topDocs;
                topDocs = indexSearcher.search(booleanQuery.build(), indexReader.numDocs());

                for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
                    IndexableField[] storableFields = indexSearcher.doc(scoreDoc.doc).getFields(textFieldName);
                    for (IndexableField singleStorableField : storableFields) {
                        if (singleStorableField != null) {
                            BytesRef text = new BytesRef(singleStorableField.stringValue());
                            generateCooccurences(text.utf8ToString(), analyzer, CooccurList, tpClass);
                        }
                    }
                }
                CooccurList.trimbyPercent(tpClass, direction, percent);

            }
        }
    }
    CooccurList.savetoFile(saveDir);
    System.out.println(":::Generation Complete:::");
}

From source file:cz.muni.fi.mias.math.MathTokenizer.java

License:Apache License

@Override
// NB: TokenStream implementation classes or at least their incrementToken() implementation must be final
public final boolean incrementToken() {
    clearAttributes();//from w  w w  . ja v  a2  s  . c o m
    if (nextIt()) {
        Formula f = itForms.next();
        termAtt.setEmpty();
        String nodeString = nodeToString(f.getNode(), false);
        // Trim node string representation to fit Lucene index term max size
        if (nodeString.length() >= TOKEN_TRIM_LENGTH) {
            LOG.warn("Node string representation too long (" + nodeString.length() + " chars), cut to "
                    + TOKEN_TRIM_LENGTH + " chars.");
            nodeString.substring(0, TOKEN_TRIM_LENGTH);
        }
        termAtt.append(nodeString);
        byte[] payload = PayloadHelper.encodeFloatToShort(f.getWeight());
        payAtt.setPayload(new BytesRef(payload));
        posAtt.setPositionIncrement(increment);
        increment = 0;
        return true;
    }
    return false;
}

From source file:cz.muni.fi.webmias.suggest.TermsListIterator.java

License:Apache License

@Override
public BytesRef next() throws IOException {
    if (iterator.hasNext()) {
        return new BytesRef(iterator.next());
    }//from  www.j  a  v a2s.c o m
    return null;
}

From source file:de.carne.filescanner.swt.main.SearchIndex.java

License:Open Source License

private void addResultHelper(Updater updater, FileScannerResult result, boolean checkForUpdate)
        throws IOException {
    BytesRef resultKey = new BytesRef(result.key());
    long resultEnd = result.end();
    Document currentDocument = (checkForUpdate ? updater.getDocument(resultKey) : null);
    boolean processResultChildren;
    boolean checkChildrenForUpdate;

    if (currentDocument == null) {
        LOG.debug("Adding result ''{0}'' to search index", result);

        updater.addDocument(buildDocument(resultKey, resultEnd, getResultContent(result)));
        processResultChildren = true;/*from w w w . j a v a2s.  co m*/
        checkChildrenForUpdate = false;
    } else if (currentDocument.getField(FIELD_END_STORED).numericValue().longValue() != resultEnd) {
        LOG.debug("Updating result ''{0}'' in search index", result);

        updater.updateDocument(resultKey, buildDocument(resultKey, resultEnd, getResultContent(result)));
        processResultChildren = true;
        checkChildrenForUpdate = true;
    } else {
        LOG.debug("Result ''{0}'' is already up-to-date in search index", result);

        processResultChildren = result.type() == FileScannerResult.Type.INPUT;
        checkChildrenForUpdate = true;
    }
    if (processResultChildren) {
        for (FileScannerResult resultChild : result.children()) {
            addResultHelper(updater, resultChild, checkChildrenForUpdate);
        }
    }
}

From source file:de.carne.filescanner.swt.main.SearchIndex.java

License:Open Source License

public byte @Nullable [] searchFoward(@Nullable FileScannerResult start, String query) throws IOException {
    return getSearcher().search((start != null ? new BytesRef(start.key()) : null), null, query, SORT_FORWARD);
}

From source file:de.carne.filescanner.swt.main.SearchIndex.java

License:Open Source License

public byte @Nullable [] searchBackward(@Nullable FileScannerResult start, String query) throws IOException {
    return getSearcher().search(null, (start != null ? new BytesRef(start.key()) : null), query, SORT_BACKWARD);
}

From source file:de.jetsli.lumeo.EdgeFilterSequenceTest.java

License:Apache License

@Test
public void testIteratorWithVertexFilter() {
    g.createAutomaticIndex("edges", Edge.class, Helper.<String>set("hello"));
    g.createAutomaticIndex("vertices", Vertex.class, Helper.<String>set("hellov"));
    Vertex x1 = g.addVertex("final");
    Vertex x2 = g.addVertex("countdown");
    Edge e = g.addEdge(null, x1, x2, "association");
    e.setProperty("hello", "test");
    e.setProperty("hellov", "test");

    Vertex x3 = g.addVertex("test");
    x3.setProperty("hello", "test");
    e = g.addEdge(null, x1, x3, "association2");
    e.setProperty("hello", "world");
    e.setProperty("hellov", "world");
    refresh();/*www .  j  a va 2s . com*/

    assertCount(1, new EdgeFilterSequence(g).setFilter(new TermFilter("hello", new BytesRef("world"))));

    // hellov gets not indexed
    assertCount(0, new EdgeFilterSequence(g).setFilter(new TermFilter("hellov", new BytesRef("world"))));

    // ... but stored
    e = new EdgeFilterSequence(g).setFilter(new TermFilter("hello", new BytesRef("world"))).next();
    assertEquals("world", e.getProperty("hellov"));
}