Example usage for org.apache.lucene.util BytesRef deepCopyOf

Introduction

In this page you can find the example usage for org.apache.lucene.util BytesRef deepCopyOf.

Prototype

public static BytesRef deepCopyOf(BytesRef other) 

Source Link

Document

Creates a new BytesRef that points to a copy of the bytes from other

The returned BytesRef will have a length of other.length and an offset of zero.

Usage

From source file:org.elasticsearch.search.MultiValueModeTests.java

License:Apache License

public void testSingleValuedStrings() throws Exception {
    final int numDocs = scaledRandomIntBetween(1, 100);
    final BytesRef[] array = new BytesRef[numDocs];
    final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
    for (int i = 0; i < array.length; ++i) {
        if (randomBoolean()) {
            array[i] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8));
            if (docsWithValue != null) {
                docsWithValue.set(i);/*  w  ww .j  av  a 2  s .  com*/
            }
        } else {
            array[i] = new BytesRef();
            if (docsWithValue != null && randomBoolean()) {
                docsWithValue.set(i);
            }
        }
    }
    final BinaryDocValues singleValues = new BinaryDocValues() {
        @Override
        public BytesRef get(int docID) {
            return BytesRef.deepCopyOf(array[docID]);
        }
    };
    final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue);
    verify(multiValues, numDocs);
    final FixedBitSet rootDocs = randomRootDocs(numDocs);
    final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
    verify(multiValues, numDocs, rootDocs, innerDocs);
}

---

From source file:org.elasticsearch.search.MultiValueModeTests.java

License:Apache License

public void testMultiValuedStrings() throws Exception {
    final int numDocs = scaledRandomIntBetween(1, 100);
    final BytesRef[][] array = new BytesRef[numDocs][];
    for (int i = 0; i < numDocs; ++i) {
        final BytesRef[] values = new BytesRef[randomInt(4)];
        for (int j = 0; j < values.length; ++j) {
            values[j] = new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8));
        }//from  w w  w.  j  av a 2 s  .com
        Arrays.sort(values);
        array[i] = values;
    }
    final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() {
        int doc;

        @Override
        public BytesRef valueAt(int index) {
            return BytesRef.deepCopyOf(array[doc][index]);
        }

        @Override
        public void setDocument(int doc) {
            this.doc = doc;
        }

        @Override
        public int count() {
            return array[doc].length;
        }
    };
    verify(multiValues, numDocs);
    final FixedBitSet rootDocs = randomRootDocs(numDocs);
    final FixedBitSet innerDocs = randomInnerDocs(rootDocs);
    verify(multiValues, numDocs, rootDocs, innerDocs);
}

---

From source file:org.elasticsearch.search.MultiValueModeTests.java

License:Apache License

private void verify(SortedBinaryDocValues values, int maxDoc) {
    for (BytesRef missingValue : new BytesRef[] { new BytesRef(),
            new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) {
        for (MultiValueMode mode : new MultiValueMode[] { MultiValueMode.MIN, MultiValueMode.MAX }) {
            final BinaryDocValues selected = mode.select(values, missingValue);
            for (int i = 0; i < maxDoc; ++i) {
                final BytesRef actual = selected.get(i);
                BytesRef expected = null;
                values.setDocument(i);//from   w  w  w  .  ja v  a2  s  .  co m
                int numValues = values.count();
                if (numValues == 0) {
                    expected = missingValue;
                } else {
                    for (int j = 0; j < numValues; ++j) {
                        if (expected == null) {
                            expected = BytesRef.deepCopyOf(values.valueAt(j));
                        } else {
                            if (mode == MultiValueMode.MIN) {
                                expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected
                                        : BytesRef.deepCopyOf(values.valueAt(j));
                            } else if (mode == MultiValueMode.MAX) {
                                expected = expected.compareTo(values.valueAt(j)) > 0 ? expected
                                        : BytesRef.deepCopyOf(values.valueAt(j));
                            }
                        }
                    }
                    if (expected == null) {
                        expected = missingValue;
                    }
                }

                assertEquals(mode.toString() + " docId=" + i, expected, actual);
            }
        }
    }
}

---

From source file:org.elasticsearch.search.MultiValueModeTests.java

License:Apache License

private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs)
        throws IOException {
    for (BytesRef missingValue : new BytesRef[] { new BytesRef(),
            new BytesRef(RandomStrings.randomAsciiOfLength(getRandom(), 8)) }) {
        for (MultiValueMode mode : new MultiValueMode[] { MultiValueMode.MIN, MultiValueMode.MAX }) {
            final BinaryDocValues selected = mode.select(values, missingValue, rootDocs,
                    new BitSetIterator(innerDocs, 0L), maxDoc);
            int prevRoot = -1;
            for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc
                    ? rootDocs.nextSetBit(root + 1)
                    : -1) {//  ww  w .j  a v a 2  s .  c o m
                final BytesRef actual = selected.get(root);
                BytesRef expected = null;
                for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1
                        && child < root; child = innerDocs.nextSetBit(child + 1)) {
                    values.setDocument(child);
                    for (int j = 0; j < values.count(); ++j) {
                        if (expected == null) {
                            expected = BytesRef.deepCopyOf(values.valueAt(j));
                        } else {
                            if (mode == MultiValueMode.MIN) {
                                expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected
                                        : BytesRef.deepCopyOf(values.valueAt(j));
                            } else if (mode == MultiValueMode.MAX) {
                                expected = expected.compareTo(values.valueAt(j)) > 0 ? expected
                                        : BytesRef.deepCopyOf(values.valueAt(j));
                            }
                        }
                    }
                }
                if (expected == null) {
                    expected = missingValue;
                }

                assertEquals(mode.toString() + " docId=" + root, expected, actual);

                prevRoot = root;
            }
        }
    }
}

---

From source file:org.elasticsearch.search.suggest.phrase.DirectCandidateGenerator.java

License:Apache License

protected void postFilter(final Candidate candidate, final CharsRef spare, BytesRef byteSpare,
        final List<Candidate> candidates) throws IOException {
    if (postFilter == null) {
        candidates.add(candidate);/*from w  ww .  j  a v a2  s  .  c  o  m*/
    } else {
        final BytesRef result = byteSpare;
        SuggestUtils.analyze(postFilter, candidate.term, field, new SuggestUtils.TokenConsumer() {
            @Override
            public void nextToken() throws IOException {
                this.fillBytesRef(result);

                if (posIncAttr.getPositionIncrement() > 0 && result.bytesEquals(candidate.term)) {
                    BytesRef term = BytesRef.deepCopyOf(result);
                    long freq = frequency(term);
                    candidates.add(new Candidate(BytesRef.deepCopyOf(term), freq, candidate.stringDistance,
                            score(candidate.frequency, candidate.stringDistance, dictSize), false));
                } else {
                    candidates.add(
                            new Candidate(BytesRef.deepCopyOf(result), candidate.frequency, nonErrorLikelihood,
                                    score(candidate.frequency, candidate.stringDistance, dictSize), false));
                }
            }
        }, spare);
    }
}

---

From source file:org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.java

License:Apache License

public Result getCorrections(TokenStream stream, final CandidateGenerator generator, float maxErrors,
        int numCorrections, IndexReader reader, WordScorer wordScorer, BytesRef separator, float confidence,
        int gramSize) throws IOException {

    final List<CandidateSet> candidateSetsList = new ArrayList<DirectCandidateGenerator.CandidateSet>();
    SuggestUtils.analyze(stream, new SuggestUtils.TokenConsumer() {
        CandidateSet currentSet = null;//  w  w w  . ja  v  a2  s . c  o  m
        private TypeAttribute typeAttribute;
        private final BytesRef termsRef = new BytesRef();
        private boolean anyUnigram = false;
        private boolean anyTokens = false;

        @Override
        public void reset(TokenStream stream) {
            super.reset(stream);
            typeAttribute = stream.addAttribute(TypeAttribute.class);
        }

        @Override
        public void nextToken() throws IOException {
            anyTokens = true;
            BytesRef term = fillBytesRef(termsRef);
            if (requireUnigram && typeAttribute.type() == ShingleFilter.DEFAULT_TOKEN_TYPE) {
                return;
            }
            anyUnigram = true;
            if (posIncAttr.getPositionIncrement() == 0 && typeAttribute.type() == SynonymFilter.TYPE_SYNONYM) {
                assert currentSet != null;
                long freq = 0;
                if ((freq = generator.frequency(term)) > 0) {
                    currentSet.addOneCandidate(
                            generator.createCandidate(BytesRef.deepCopyOf(term), freq, realWordLikelihood));
                }
            } else {
                if (currentSet != null) {
                    candidateSetsList.add(currentSet);
                }
                currentSet = new CandidateSet(Candidate.EMPTY,
                        generator.createCandidate(BytesRef.deepCopyOf(term), true));
            }
        }

        @Override
        public void end() {
            if (currentSet != null) {
                candidateSetsList.add(currentSet);
            }
            if (requireUnigram && !anyUnigram && anyTokens) {
                throw new IllegalStateException("At least one unigram is required but all tokens were ngrams");
            }
        }
    });

    if (candidateSetsList.isEmpty() || candidateSetsList.size() >= tokenLimit) {
        return Result.EMPTY;
    }

    for (CandidateSet candidateSet : candidateSetsList) {
        generator.drawCandidates(candidateSet);
    }
    double cutoffScore = Double.MIN_VALUE;
    CandidateScorer scorer = new CandidateScorer(wordScorer, numCorrections, gramSize);
    CandidateSet[] candidateSets = candidateSetsList.toArray(new CandidateSet[candidateSetsList.size()]);
    if (confidence > 0.0) {
        Candidate[] candidates = new Candidate[candidateSets.length];
        for (int i = 0; i < candidates.length; i++) {
            candidates[i] = candidateSets[i].originalTerm;
        }
        double inputPhraseScore = scorer.score(candidates, candidateSets);
        cutoffScore = inputPhraseScore * confidence;
    }
    Correction[] findBestCandiates = scorer.findBestCandiates(candidateSets, maxErrors, cutoffScore);

    return new Result(findBestCandiates, cutoffScore);
}

---

From source file:org.elasticsearch.search.suggest.term.TermSuggester.java

License:Apache License

private List<Token> queryTerms(SuggestionContext suggestion, CharsRef spare) throws IOException {
    final List<Token> result = new ArrayList<TermSuggester.Token>();
    final String field = suggestion.getField();
    SuggestUtils.analyze(suggestion.getAnalyzer(), suggestion.getText(), field,
            new SuggestUtils.TokenConsumer() {
                @Override/*from w w w.j  a v a 2s  .c o m*/
                public void nextToken() {
                    Term term = new Term(field, BytesRef.deepCopyOf(fillBytesRef(new BytesRef())));
                    result.add(new Token(term, offsetAttr.startOffset(), offsetAttr.endOffset()));
                }
            }, spare);
    return result;
}

---

From source file:org.elasticsearch.test.unit.index.mapper.geo.LatLonMappingGeoPointTests.java

License:Apache License

@Test
public void testLatLonValuesStored() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes")
            .endObject().endObject().endObject().endObject().string();

    DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);

    ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
            .startObject("point").field("lat", 1.2).field("lon", 1.3).endObject().endObject().bytes());

    assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lat").binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.2)));
    assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lon").binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.3)));
    assertThat(doc.rootDoc().getField("point.geohash"), nullValue());
    assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
}

---

From source file:org.elasticsearch.test.unit.index.mapper.geo.LatLonMappingGeoPointTests.java

License:Apache License

@Test
public void testArrayLatLonValues() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes")
            .endObject().endObject().endObject().endObject().string();

    DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);

    ParsedDocument doc = defaultMapper.parse("type", "1",
            XContentFactory.jsonBuilder().startObject().startArray("point").startObject().field("lat", 1.2)
                    .field("lon", 1.3).endObject().startObject().field("lat", 1.4).field("lon", 1.5).endObject()
                    .endArray().endObject().bytes());

    assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
    assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[0].binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.2)));
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[0].binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.3)));
    assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lat")[1].binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.4)));
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getFields("point.lon")[1].binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.5)));
    assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
}

---

From source file:org.elasticsearch.test.unit.index.mapper.geo.LatLonMappingGeoPointTests.java

License:Apache License

@Test
public void testLatLonInOneValueStored() throws Exception {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
            .startObject("point").field("type", "geo_point").field("lat_lon", true).field("store", "yes")
            .endObject().endObject().endObject().endObject().string();

    DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);

    ParsedDocument doc = defaultMapper.parse("type", "1",
            XContentFactory.jsonBuilder().startObject().field("point", "1.2,1.3").endObject().bytes());

    assertThat(doc.rootDoc().getField("point.lat"), notNullValue());
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lat").binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.2)));
    assertThat(doc.rootDoc().getField("point.lon"), notNullValue());
    assertThat(BytesRef.deepCopyOf(doc.rootDoc().getField("point.lon").binaryValue()).bytes,
            equalTo(Numbers.doubleToBytes(1.3)));
    assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• »