List of usage examples for org.apache.lucene.util BytesRef equals
@Override
public boolean equals(Object other)
From source file:io.crate.expression.scalar.TimeZoneParser.java
License:Apache License
public static DateTimeZone parseTimeZone(BytesRef timezone) throws IllegalArgumentException { if (timezone == null) { throw new IllegalArgumentException("invalid time zone value NULL"); }//from w ww . j av a 2s . c o m if (timezone.equals(DEFAULT_TZ_BYTES_REF)) { return DEFAULT_TZ; } DateTimeZone tz = TIME_ZONE_MAP.get(timezone); if (tz == null) { try { String text = timezone.utf8ToString(); int index = text.indexOf(':'); if (index != -1) { int beginIndex = text.charAt(0) == '+' ? 1 : 0; // format like -02:30 tz = DateTimeZone.forOffsetHoursMinutes(Integer.parseInt(text.substring(beginIndex, index)), Integer.parseInt(text.substring(index + 1))); } else { // id, listed here: http://joda-time.sourceforge.net/timezones.html // or here: http://www.joda.org/joda-time/timezones.html tz = DateTimeZone.forID(text); } } catch (IllegalArgumentException e) { throw new IllegalArgumentException( String.format(Locale.ENGLISH, "invalid time zone value '%s'", timezone.utf8ToString())); } TIME_ZONE_MAP.putIfAbsent(timezone, tz); } return tz; }
From source file:org.apache.solr.handler.component.TermsComponent.java
License:Apache License
@Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (!params.getBool(TermsParams.TERMS, false)) return;/*from w w w . j av a2 s .co m*/ String[] fields = params.getParams(TermsParams.TERMS_FIELD); NamedList<Object> termsResult = new SimpleOrderedMap<Object>(); rb.rsp.add("terms", termsResult); if (fields == null || fields.length == 0) return; int limit = params.getInt(TermsParams.TERMS_LIMIT, 10); if (limit < 0) { limit = Integer.MAX_VALUE; } String lowerStr = params.get(TermsParams.TERMS_LOWER); String upperStr = params.get(TermsParams.TERMS_UPPER); boolean upperIncl = params.getBool(TermsParams.TERMS_UPPER_INCLUSIVE, false); boolean lowerIncl = params.getBool(TermsParams.TERMS_LOWER_INCLUSIVE, true); boolean sort = !TermsParams.TERMS_SORT_INDEX .equals(params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT)); int freqmin = params.getInt(TermsParams.TERMS_MINCOUNT, 1); int freqmax = params.getInt(TermsParams.TERMS_MAXCOUNT, UNLIMITED_MAX_COUNT); if (freqmax < 0) { freqmax = Integer.MAX_VALUE; } String prefix = params.get(TermsParams.TERMS_PREFIX_STR); String regexp = params.get(TermsParams.TERMS_REGEXP_STR); Pattern pattern = regexp != null ? Pattern.compile(regexp, resolveRegexpFlags(params)) : null; boolean raw = params.getBool(TermsParams.TERMS_RAW, false); final AtomicReader indexReader = rb.req.getSearcher().getAtomicReader(); Fields lfields = indexReader.fields(); for (String field : fields) { NamedList<Integer> fieldTerms = new NamedList<Integer>(); termsResult.add(field, fieldTerms); Terms terms = lfields == null ? null : lfields.terms(field); if (terms == null) { // no terms for this field continue; } FieldType ft = raw ? null : rb.req.getSchema().getFieldTypeNoEx(field); if (ft == null) ft = new StrField(); // prefix must currently be text BytesRef prefixBytes = prefix == null ? null : new BytesRef(prefix); BytesRef upperBytes = null; if (upperStr != null) { upperBytes = new BytesRef(); ft.readableToIndexed(upperStr, upperBytes); } BytesRef lowerBytes; if (lowerStr == null) { // If no lower bound was specified, use the prefix lowerBytes = prefixBytes; } else { lowerBytes = new BytesRef(); if (raw) { // TODO: how to handle binary? perhaps we don't for "raw"... or if the field exists // perhaps we detect if the FieldType is non-character and expect hex if so? lowerBytes = new BytesRef(lowerStr); } else { lowerBytes = new BytesRef(); ft.readableToIndexed(lowerStr, lowerBytes); } } TermsEnum termsEnum = terms.iterator(null); BytesRef term = null; if (lowerBytes != null) { if (termsEnum.seekCeil(lowerBytes) == TermsEnum.SeekStatus.END) { termsEnum = null; } else { term = termsEnum.term(); //Only advance the enum if we are excluding the lower bound and the lower Term actually matches if (lowerIncl == false && term.equals(lowerBytes)) { term = termsEnum.next(); } } } else { // position termsEnum on first term term = termsEnum.next(); } int i = 0; BoundedTreeSet<CountPair<BytesRef, Integer>> queue = (sort ? new BoundedTreeSet<CountPair<BytesRef, Integer>>(limit) : null); CharsRef external = new CharsRef(); while (term != null && (i < limit || sort)) { boolean externalized = false; // did we fill in "external" yet for this term? // stop if the prefix doesn't match if (prefixBytes != null && !StringHelper.startsWith(term, prefixBytes)) break; if (pattern != null) { // indexed text or external text? // TODO: support "raw" mode? ft.indexedToReadable(term, external); externalized = true; if (!pattern.matcher(external).matches()) { term = termsEnum.next(); continue; } } if (upperBytes != null) { int upperCmp = term.compareTo(upperBytes); // if we are past the upper term, or equal to it (when don't include upper) then stop. if (upperCmp > 0 || (upperCmp == 0 && !upperIncl)) break; } // This is a good term in the range. Check if mincount/maxcount conditions are satisfied. int docFreq = termsEnum.docFreq(); if (docFreq >= freqmin && docFreq <= freqmax) { // add the term to the list if (sort) { queue.add(new CountPair<BytesRef, Integer>(BytesRef.deepCopyOf(term), docFreq)); } else { // TODO: handle raw somehow if (!externalized) { ft.indexedToReadable(term, external); } fieldTerms.add(external.toString(), docFreq); i++; } } term = termsEnum.next(); } if (sort) { for (CountPair<BytesRef, Integer> item : queue) { if (i >= limit) break; ft.indexedToReadable(item.key, external); fieldTerms.add(external.toString(), item.val); i++; } } } }
From source file:org.elasticsearch.index.fielddata.DuelFieldDataTests.java
License:Apache License
private static void duelFieldDataBytes(Random random, AtomicReaderContext context, IndexFieldData<?> left, IndexFieldData<?> right, Preprocessor pre) throws Exception { AtomicFieldData<?> leftData = random.nextBoolean() ? left.load(context) : left.loadDirect(context); AtomicFieldData<?> rightData = random.nextBoolean() ? right.load(context) : right.loadDirect(context); assertThat(leftData.getNumDocs(), equalTo(rightData.getNumDocs())); int numDocs = leftData.getNumDocs(); BytesValues leftBytesValues = leftData.getBytesValues(random.nextBoolean()); BytesValues rightBytesValues = rightData.getBytesValues(random.nextBoolean()); BytesRef leftSpare = new BytesRef(); BytesRef rightSpare = new BytesRef(); for (int i = 0; i < numDocs; i++) { int numValues = 0; assertThat((numValues = leftBytesValues.setDocument(i)), equalTo(rightBytesValues.setDocument(i))); BytesRef previous = null;/*from ww w. j a v a 2s.co m*/ for (int j = 0; j < numValues; j++) { rightSpare.copyBytes(rightBytesValues.nextValue()); leftSpare.copyBytes(leftBytesValues.nextValue()); assertThat(rightSpare.hashCode(), equalTo(rightBytesValues.currentValueHash())); assertThat(leftSpare.hashCode(), equalTo(leftBytesValues.currentValueHash())); if (previous != null && leftBytesValues.getOrder() == rightBytesValues.getOrder()) { // we can only compare the assertThat(pre.compare(previous, rightSpare), lessThan(0)); } previous = BytesRef.deepCopyOf(rightSpare); pre.toString(rightSpare); pre.toString(leftSpare); assertThat(pre.toString(leftSpare), equalTo(pre.toString(rightSpare))); if (leftSpare.equals(rightSpare)) { assertThat(leftBytesValues.currentValueHash(), equalTo(rightBytesValues.currentValueHash())); } } } }
From source file:org.elasticsearch.search.aggregations.bucket.composite.GlobalOrdinalValuesSource.java
License:Apache License
@Override
LeafBucketCollector getLeafCollector(Comparable<?> value, LeafReaderContext context, LeafBucketCollector next)
throws IOException {
if (value.getClass() != BytesRef.class) {
throw new IllegalArgumentException("Expected BytesRef, got " + value.getClass());
}// w w w.java 2 s. c o m
BytesRef term = (BytesRef) value;
final SortedSetDocValues dvs = docValuesFunc.apply(context);
if (lookup == null) {
initLookup(dvs);
}
return new LeafBucketCollector() {
boolean currentValueIsSet = false;
@Override
public void collect(int doc, long bucket) throws IOException {
if (!currentValueIsSet) {
if (dvs.advanceExact(doc)) {
long ord;
while ((ord = dvs.nextOrd()) != NO_MORE_ORDS) {
if (term.equals(lookup.lookupOrd(ord))) {
currentValueIsSet = true;
currentValue = ord;
break;
}
}
}
}
assert currentValueIsSet;
next.collect(doc, bucket);
}
};
}
From source file:org.elasticsearch.search.facet.terms.strings.TermsStringOrdinalsFacetExecutor.java
License:Apache License
@Override public InternalFacet buildFacet(String facetName) { final CharsRef spare = new CharsRef(); AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size()); for (ReaderAggregator aggregator : aggregators) { if (aggregator.nextPosition()) { queue.add(aggregator);//from ww w . jav a 2 s . c o m } } // YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes if (shardSize < EntryPriorityQueue.LIMIT) { // optimize to use priority size EntryPriorityQueue ordered = new EntryPriorityQueue(shardSize, comparatorType.comparator()); while (queue.size() > 0) { ReaderAggregator agg = queue.top(); BytesRef value = agg.copyCurrent(); // we need to makeSafe it, since we end up pushing it... (can we get around this?) int count = 0; do { count += agg.counts.get(agg.position); if (agg.nextPosition()) { agg = queue.updateTop(); } else { // we are done with this reader queue.pop(); agg = queue.top(); } } while (agg != null && value.equals(agg.current)); if (count > minCount) { if (excluded != null && excluded.contains(value)) { continue; } if (matcher != null) { UnicodeUtil.UTF8toUTF16(value, spare); assert spare.toString().equals(value.utf8ToString()); if (!matcher.reset(spare).matches()) { continue; } } InternalStringTermsFacet.TermEntry entry = new InternalStringTermsFacet.TermEntry(value, count); ordered.insertWithOverflow(entry); } } InternalStringTermsFacet.TermEntry[] list = new InternalStringTermsFacet.TermEntry[ordered.size()]; for (int i = ordered.size() - 1; i >= 0; i--) { list[i] = (InternalStringTermsFacet.TermEntry) ordered.pop(); } return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total); } BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered = new BoundedTreeSet<InternalStringTermsFacet.TermEntry>( comparatorType.comparator(), shardSize); while (queue.size() > 0) { ReaderAggregator agg = queue.top(); BytesRef value = agg.copyCurrent(); // we need to makeSafe it, since we end up pushing it... (can we work around that?) int count = 0; do { count += agg.counts.get(agg.position); if (agg.nextPosition()) { agg = queue.updateTop(); } else { // we are done with this reader queue.pop(); agg = queue.top(); } } while (agg != null && value.equals(agg.current)); if (count > minCount) { if (excluded != null && excluded.contains(value)) { continue; } if (matcher != null) { UnicodeUtil.UTF8toUTF16(value, spare); assert spare.toString().equals(value.utf8ToString()); if (!matcher.reset(spare).matches()) { continue; } } InternalStringTermsFacet.TermEntry entry = new InternalStringTermsFacet.TermEntry(value, count); ordered.add(entry); } } return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, missing, total); }
From source file:org.elasticsearch.test.unit.index.fielddata.DuellFieldDataTest.java
License:Apache License
private static void duellFieldDataBytes(Random random, AtomicReaderContext context, IndexFieldData left, IndexFieldData right, Preprocessor pre) throws Exception { AtomicFieldData leftData = random.nextBoolean() ? left.load(context) : left.loadDirect(context); AtomicFieldData rightData = random.nextBoolean() ? right.load(context) : right.loadDirect(context); assertThat(leftData.isMultiValued(), equalTo(rightData.isMultiValued())); assertThat(leftData.getNumDocs(), equalTo(rightData.getNumDocs())); int numDocs = leftData.getNumDocs(); BytesValues leftBytesValues = random.nextBoolean() ? leftData.getBytesValues() : leftData.getHashedBytesValues(); BytesValues rightBytesValues = random.nextBoolean() ? rightData.getBytesValues() : rightData.getHashedBytesValues(); for (int i = 0; i < numDocs; i++) { assertThat(leftBytesValues.hasValue(i), equalTo(rightBytesValues.hasValue(i))); if (leftBytesValues.hasValue(i)) { assertThat(pre.toString(leftBytesValues.getValue(i)), equalTo(pre.toString(rightBytesValues.getValue(i)))); } else {/*from w w w . j a va 2s . co m*/ assertThat(leftBytesValues.getValue(i), nullValue()); assertThat(rightBytesValues.getValue(i), nullValue()); } boolean hasValue = leftBytesValues.hasValue(i); Iter leftIter = leftBytesValues.getIter(i); Iter rightIter = rightBytesValues.getIter(i); assertThat(leftIter.hasNext(), equalTo(rightIter.hasNext())); assertThat(leftIter.hasNext(), equalTo(hasValue)); while (leftIter.hasNext()) { assertThat(hasValue, equalTo(true)); assertThat(leftIter.hasNext(), equalTo(rightIter.hasNext())); BytesRef rightBytes = rightIter.next(); BytesRef leftBytes = leftIter.next(); assertThat(pre.toString(leftBytes), equalTo(pre.toString(rightBytes))); if (rightBytes.equals(leftBytes)) { assertThat(leftIter.hash(), equalTo(rightIter.hash()));// call twice assertThat(leftIter.hash(), equalTo(rightIter.hash())); assertThat(leftIter.hash(), equalTo(rightBytes.hashCode())); assertThat(rightIter.hash(), equalTo(leftBytes.hashCode())); } } assertThat(leftIter.hasNext(), equalTo(rightIter.hasNext())); } }
From source file:org.fastcatsearch.ir.search.DefaultRanker.java
License:Apache License
@Override public boolean push(HitElement e) { if (e.getBundleKey() != null) { BytesRef bundleKey = e.getBundleKey(); for (int i = 1; i <= size; i++) { if (bundleKey.equals(((HitElement) heap[i]).getBundleKey())) { /*/* www. j a v a2s . c o m*/ * ?? bundle ? ?? ? ? ?. */ if (compare(e, (HitElement) heap[i]) < 0) { // ? . // . replaceEl(i, e); // break; return true; } // logger.debug("Do no push > {}", e.docNo()); return false; } } } //bundle? ?? bundle? push. return super.push(e); }
From source file:org.fastcatsearch.ir.search.HitMerger.java
License:Apache License
@Override public boolean push(FixedHitReader e) { if (e.read().getBundleKey() != null) { BytesRef bundleKey = e.read().getBundleKey(); for (int i = 1; i <= size; i++) { if (bundleKey.equals(((HitElement) ((FixedHitReader) heap[i]).read()).getBundleKey())) { //?? bundle ? ? push . logger.debug("?? Bundle found!!!"); return false; }// w w w .ja v a 2 s . co m } } //bundle? ?? bundle? push. return super.push(e); }
From source file:org.fastcatsearch.ir.search.HitRanker.java
License:Apache License
@Override public boolean push(HitElement e) { if (e.getBundleKey() != null) { BytesRef bundleKey = e.getBundleKey(); for (int i = 1; i <= size; i++) { if (bundleKey.equals(((HitElement) heap[i]).getBundleKey())) { /*/*w w w .java 2 s . c o m*/ * ?? bundle ? ?? ? ? ?. */ if (compare(e, (HitElement) heap[i]) < 0) { // ? . // . replaceEl(i, e); // break; return true; } // logger.debug("Do no push > {}", e.docNo()); return false; } } } // logger.debug("Continue to push > {}", e.docNo()); // bundle? ?? bundle? push. return super.push(e); }