Example usage for org.apache.lucene.util BytesRefBuilder copyChars

Introduction

In this page you can find the example usage for org.apache.lucene.util BytesRefBuilder copyChars.

Prototype

public void copyChars(CharSequence text) 

Source Link

Document

Replace the content of this buffer with UTF-8 encoded bytes that would represent the provided text.

Usage

From source file:com.meizu.nlp.classification.BooleanPerceptronClassifier.java

License:Apache License

private void updateFST(SortedMap<String, Double> weights) throws IOException {
    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
    Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs);
    BytesRefBuilder scratchBytes = new BytesRefBuilder();
    IntsRefBuilder scratchInts = new IntsRefBuilder();
    for (Map.Entry<String, Double> entry : weights.entrySet()) {
        scratchBytes.copyChars(entry.getKey());
        fstBuilder.add(Util.toIntsRef(scratchBytes.get(), scratchInts), entry.getValue().longValue());
    }/* www.  j a v a 2 s .c  o m*/
    fst = fstBuilder.finish();
}

---

From source file:edu.upenn.library.solrplugins.CaseInsensitiveSortingTextField.java

License:Apache License

@Override
public BytesRef normalizeQueryTarget(String val, boolean strict, String fieldName, boolean appendExtraDelim)
        throws IOException {
    TokenStream ts = getQueryAnalyzer().tokenStream(fieldName, val);
    try {/*from  ww w.j  av a 2  s.  c o m*/
        ts.reset();
        CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
        TypeAttribute typeAtt = ts.getAttribute(TypeAttribute.class);
        String matchType = strict ? INDEXED_TOKEN_TYPE : NORMALIZED_TOKEN_TYPE;
        while (ts.incrementToken()) {
            if (matchType.equals(typeAtt.type())) {
                BytesRefBuilder ret = new BytesRefBuilder();
                ret.copyChars(termAtt.toString());
                if (!strict || appendExtraDelim) {
                    ret.append(delimBytes, 0, delimBytes.length);
                }
                return ret.get();
            }
        }
        return new BytesRef(BytesRef.EMPTY_BYTES);
    } finally {
        ts.close();
    }
}

---

From source file:examples.fst.FstTest.java

public static void main(String[] args) throws IOException {
    // Input values (keys). These must be provided to Builder in Unicode sorted order!
    String inputValues[] = { "cat", "dog", "dogs" };
    long outputValues[] = { 5, 7, 12 };

    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
    Builder<Long> builder = new Builder<Long>(INPUT_TYPE.BYTE1, outputs);
    BytesRefBuilder scratchBytes = new BytesRefBuilder();
    IntsRefBuilder scratchInts = new IntsRefBuilder();
    for (int i = 0; i < inputValues.length; i++) {
        scratchBytes.copyChars(inputValues[i]);
        builder.add(Util.toIntsRef(scratchBytes.toBytesRef(), scratchInts), outputValues[i]);
    }// w  ww  .  j a  va2s  .  c o m
    FST<Long> fst = builder.finish();

    Long value = Util.get(fst, new BytesRef("dog"));
    System.out.println(value); // 7

    // Only works because outputs are also in sorted order
    IntsRef key = Util.getByOutput(fst, 12);
    System.out.println(Util.toBytesRef(key, scratchBytes).utf8ToString()); // dogs

}

---

From source file:org.apache.solr.analytics.util.valuesource.MultiStringFunction.java

License:Apache License

@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
    final FunctionValues[] valsArr = new FunctionValues[sources.length];
    for (int i = 0; i < sources.length; i++) {
        valsArr[i] = sources[i].getValues(context, readerContext);
    }//ww  w . ja v  a 2  s  .co m

    return new StrDocValues(this) {
        @Override
        public String strVal(int doc) {
            CharSequence cs = func(doc, valsArr);
            return cs != null ? cs.toString() : null;
        }

        @Override
        public boolean exists(int doc) {
            boolean exists = true;
            for (FunctionValues val : valsArr) {
                exists = exists & val.exists(doc);
            }
            return exists;
        }

        @Override
        public boolean bytesVal(int doc, BytesRefBuilder bytes) {
            bytes.clear();
            CharSequence cs = func(doc, valsArr);
            if (cs != null) {
                bytes.copyChars(func(doc, valsArr));
                return true;
            } else {
                return false;
            }
        }

        @Override
        public String toString(int doc) {
            StringBuilder sb = new StringBuilder();
            sb.append(name()).append('(');
            boolean firstTime = true;
            for (FunctionValues vals : valsArr) {
                if (firstTime) {
                    firstTime = false;
                } else {
                    sb.append(',');
                }
                sb.append(vals.toString(doc));
            }
            sb.append(')');
            return sb.toString();
        }

        @Override
        public ValueFiller getValueFiller() {
            return new ValueFiller() {
                private final MutableValueStr mval = new MutableValueStr();

                @Override
                public MutableValue getValue() {
                    return mval;
                }

                @Override
                public void fillValue(int doc) {
                    mval.exists = bytesVal(doc, mval.value);
                }
            };
        }
    };
}

---

From source file:org.apache.solr.analytics.util.valuesource.SingleStringFunction.java

License:Apache License

@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
    final FunctionValues vals = source.getValues(context, readerContext);
    return new StrDocValues(this) {
        @Override//from  ww  w.  j av  a 2 s.  c  om
        public String strVal(int doc) {
            CharSequence cs = func(doc, vals);
            return cs != null ? cs.toString() : null;
        }

        @Override
        public boolean bytesVal(int doc, BytesRefBuilder bytes) {
            CharSequence cs = func(doc, vals);
            if (cs != null) {
                bytes.copyChars(func(doc, vals));
                return true;
            } else {
                bytes.clear();
                return false;
            }
        }

        @Override
        public Object objectVal(int doc) {
            return strVal(doc);
        }

        @Override
        public boolean exists(int doc) {
            return vals.exists(doc);
        }

        @Override
        public String toString(int doc) {
            return name() + '(' + strVal(doc) + ')';
        }

        @Override
        public ValueFiller getValueFiller() {
            return new ValueFiller() {
                private final MutableValueStr mval = new MutableValueStr();

                @Override
                public MutableValue getValue() {
                    return mval;
                }

                @Override
                public void fillValue(int doc) {
                    mval.exists = bytesVal(doc, mval.value);
                }
            };
        }
    };
}

---

From source file:org.apache.solr.search.function.MultiStringFunction.java

License:Apache License

@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
    final FunctionValues[] valsArr = new FunctionValues[sources.length];
    for (int i = 0; i < sources.length; i++) {
        valsArr[i] = sources[i].getValues(context, readerContext);
    }//from w  w  w .jav a2s  .  com

    return new StrDocValues(this) {
        @Override
        public String strVal(int doc) throws IOException {
            CharSequence cs = func(doc, valsArr);
            return cs != null ? cs.toString() : null;
        }

        @Override
        public boolean exists(int doc) throws IOException {
            boolean exists = true;
            for (FunctionValues val : valsArr) {
                exists = exists & val.exists(doc);
            }
            return exists;
        }

        @Override
        public boolean bytesVal(int doc, BytesRefBuilder bytes) throws IOException {
            bytes.clear();
            CharSequence cs = func(doc, valsArr);
            if (cs != null) {
                bytes.copyChars(func(doc, valsArr));
                return true;
            } else {
                return false;
            }
        }

        @Override
        public String toString(int doc) throws IOException {
            StringBuilder sb = new StringBuilder();
            sb.append(name()).append('(');
            boolean firstTime = true;
            for (FunctionValues vals : valsArr) {
                if (firstTime) {
                    firstTime = false;
                } else {
                    sb.append(',');
                }
                sb.append(vals.toString(doc));
            }
            sb.append(')');
            return sb.toString();
        }

        @Override
        public ValueFiller getValueFiller() {
            return new ValueFiller() {
                private final MutableValueStr mval = new MutableValueStr();

                @Override
                public MutableValue getValue() {
                    return mval;
                }

                @Override
                public void fillValue(int doc) throws IOException {
                    mval.exists = bytesVal(doc, mval.value);
                }
            };
        }
    };
}

---

From source file:org.apache.solr.search.GraphTermsQParserPlugin.java

License:Apache License

@Override
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
    return new QParser(qstr, localParams, params, req) {
        @Override/*  w ww .  j a v a2  s.  co m*/
        public Query parse() throws SyntaxError {
            String fname = localParams.get(QueryParsing.F);
            FieldType ft = req.getSchema().getFieldTypeNoEx(fname);
            int maxDocFreq = localParams.getInt("maxDocFreq", Integer.MAX_VALUE);
            String qstr = localParams.get(QueryParsing.V);//never null

            if (qstr.length() == 0) {
                return new MatchNoDocsQuery();
            }

            final String[] splitVals = qstr.split(",");

            Term[] terms = new Term[splitVals.length];
            BytesRefBuilder term = new BytesRefBuilder();
            for (int i = 0; i < splitVals.length; i++) {
                String stringVal = splitVals[i].trim();
                if (ft != null) {
                    ft.readableToIndexed(stringVal, term);
                } else {
                    term.copyChars(stringVal);
                }
                BytesRef ref = term.toBytesRef();
                terms[i] = new Term(fname, ref);
            }

            ArrayUtil.timSort(terms);
            return new ConstantScoreQuery(new GraphTermsQuery(fname, terms, maxDocFreq));
        }
    };
}

---

From source file:org.codelibs.elasticsearch.common.lucene.BytesRefs.java

License:Apache License

public static BytesRef toBytesRef(Object value, BytesRefBuilder spare) {
    if (value == null) {
        return null;
    }/*from   ww w .ja va 2s  .  c o  m*/
    if (value instanceof BytesRef) {
        return (BytesRef) value;
    }
    spare.copyChars(value.toString());
    return spare.get();
}

---

From source file:org.codelibs.elasticsearch.common.Strings.java

License:Apache License

public static byte[] toUTF8Bytes(CharSequence charSequence, BytesRefBuilder spare) {
    spare.copyChars(charSequence);
    return Arrays.copyOf(spare.bytes(), spare.length());
}

---

From source file:org.codelibs.elasticsearch.index.query.TermsQueryBuilder.java

License:Apache License

/**
 * Convert the list in a way that optimizes storage in the case that all
 * elements are either integers or {String}s/{BytesRef}s. This
 * is useful to help garbage collections for use-cases that involve sending
 * very large terms queries to Elasticsearch. If the list does not only
 * contain integers or {String}s, then a list is returned where all
 * {String}s have been replaced with {BytesRef}s.
 *//*  ww w . j  av  a2 s.c om*/
static List<?> convert(List<?> list) {
    if (list.isEmpty()) {
        return Collections.emptyList();
    }

    final boolean allNumbers = list.stream().allMatch(o -> o != null && INTEGER_TYPES.contains(o.getClass()));
    if (allNumbers) {
        final long[] elements = list.stream().mapToLong(o -> ((Number) o).longValue()).toArray();
        return new AbstractList<Object>() {
            @Override
            public Object get(int index) {
                return elements[index];
            }

            @Override
            public int size() {
                return elements.length;
            }
        };
    }

    final boolean allStrings = list.stream().allMatch(o -> o != null && STRING_TYPES.contains(o.getClass()));
    if (allStrings) {
        final BytesRefBuilder builder = new BytesRefBuilder();
        try (final BytesStreamOutput bytesOut = new BytesStreamOutput()) {
            final int[] endOffsets = new int[list.size()];
            int i = 0;
            for (Object o : list) {
                BytesRef b;
                if (o instanceof BytesRef) {
                    b = (BytesRef) o;
                } else {
                    builder.copyChars(o.toString());
                    b = builder.get();
                }
                bytesOut.writeBytes(b.bytes, b.offset, b.length);
                if (i == 0) {
                    endOffsets[0] = b.length;
                } else {
                    endOffsets[i] = Math.addExact(endOffsets[i - 1], b.length);
                }
                ++i;
            }
            final BytesReference bytes = bytesOut.bytes();
            return new AbstractList<Object>() {
                @Override
                public Object get(int i) {
                    final int startOffset = i == 0 ? 0 : endOffsets[i - 1];
                    final int endOffset = endOffsets[i];
                    return bytes.slice(startOffset, endOffset - startOffset).toBytesRef();
                }

                @Override
                public int size() {
                    return endOffsets.length;
                }
            };
        }
    }

    return list.stream().map(o -> o instanceof String ? new BytesRef(o.toString()) : o)
            .collect(Collectors.toList());
}

---

• «
• 1
• 2
• »