Java tutorial
/***************************************************************************** * * Copyright (C) Zenoss, Inc. 2014, all rights reserved. * * This content is made available according to terms specified in * License.zenoss under the directory where your Zenoss product is installed. * ****************************************************************************/ package org.zenoss.zep.index.impl.solr; import com.google.common.collect.*; import com.google.protobuf.ProtocolMessageEnum; import org.springframework.util.StringUtils; import org.zenoss.protobufs.util.Util.TimestampRange; import org.zenoss.protobufs.zep.Zep.EventDetailItem; import org.zenoss.protobufs.zep.Zep.EventDetailItem.EventDetailType; import org.zenoss.zep.ZepException; import org.zenoss.zep.index.IndexedDetailsConfiguration; import org.zenoss.zep.index.impl.BaseQueryBuilder; import org.zenoss.zep.index.impl.IndexConstants; import org.zenoss.zep.utils.IpRange; import org.zenoss.zep.utils.IpUtils; import java.net.InetAddress; import java.util.*; import java.util.Map.Entry; public class SolrQueryBuilder extends BaseQueryBuilder<SolrQueryBuilder> { private final IndexedDetailsConfiguration indexedDetailsConfiguration; private transient List<String> clauses; public SolrQueryBuilder(IndexedDetailsConfiguration configuration) { this(Occur.MUST, configuration); } public SolrQueryBuilder(Occur occur, IndexedDetailsConfiguration configuration) { super(occur); this.indexedDetailsConfiguration = configuration; } @Override protected SolrQueryBuilder subBuilder(Occur occur) { return new SolrQueryBuilder(occur, indexedDetailsConfiguration); } @Override protected EventDetailType getDetailType(String key) throws ZepException { EventDetailItem item = indexedDetailsConfiguration.getEventDetailItemsByName().get(key); return (item == null) ? null : item.getType(); } @Override protected String getDetailKey(String key) throws ZepException { EventDetailItem item = indexedDetailsConfiguration.getEventDetailItemsByName().get(key); if (item == null) return null; switch (item.getType()) { case STRING: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_s"; case INTEGER: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_i"; case FLOAT: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_f"; case LONG: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_l"; case DOUBLE: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_d"; case IP_ADDRESS: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_ip"; case PATH: return SolrEventIndexMapper.DETAIL_INDEX_PREFIX + item.getKey() + "_path"; default: throw new IllegalStateException("Unexpected type: " + item.getType()); } } public synchronized String build() { this.clauses = Lists.newArrayList(); for (Entry<String, Set<?>> field : fieldsValues(FieldType.DATE_RANGE)) useDateRange(field.getKey(), (Set<TimestampRange>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.ENUM_NUMBER)) useEnumNumber(field.getKey(), (Set<ProtocolMessageEnum>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.FULL_TEXT)) useFullText(field.getKey(), (Set<String>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.IDENTIFIER)) useIdentifier(field.getKey(), (Set<String>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.IP_ADDRESS_SUBSTRING)) useIpAddressSubstring(field.getKey(), (Set<String>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.IP_ADDRESS)) useIpAddress(field.getKey(), (Set<InetAddress>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.IP_ADDRESS_RANGE)) useIpAddressRange(field.getKey(), (Set<IpRange>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.NUMERIC_RANGE)) // Might not be Integers, but Java does type-erasure, so who cares. useNumericRange(field.getKey(), (Set<Range<Integer>>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.PATH)) usePath(field.getKey(), (Set<String>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.TERM)) useTerm(field.getKey(), (Set<String>) field.getValue()); for (Entry<String, Set<?>> field : fieldsValues(FieldType.WILDCARD)) useWildcard(field.getKey(), (Set<String>) field.getValue()); for (SolrQueryBuilder sub : subClauses) { String clause = sub.build(); if (clause != null) clauses.add(clause); } if (clauses.isEmpty()) return null; final String junction = (occur == Occur.SHOULD) ? " OR " : " AND "; final StringBuilder sb = new StringBuilder(); int i = 0; for (String clause : clauses) { if (i++ > 0) sb.append(junction); if (occur == Occur.MUST_NOT) sb.append("NOT ("); else sb.append("("); sb.append(clause); sb.append(")"); } this.clauses = null; return sb.toString(); } private void useDateRange(String fieldName, Set<TimestampRange> ranges) { if (ranges == null || ranges.isEmpty()) return; for (TimestampRange range : ranges) { if (range.hasStartTime() || range.hasEndTime()) { clauses.add(rangeQuery(fieldName, range.hasStartTime() ? range.getStartTime() : null, range.hasEndTime() ? range.getEndTime() : null)); } } } private void useEnumNumber(String fieldName, Set<ProtocolMessageEnum> enums) { if (enums == null || enums.isEmpty()) return; List<Integer> values = new ArrayList<Integer>(enums.size()); for (ProtocolMessageEnum e : enums) values.add(e.getNumber()); // Condense adjacent values into one range Collections.sort(values); Iterator<Integer> it = values.iterator(); Integer from = it.next(); Integer to = from; while (it.hasNext()) { int value = it.next(); if (value == to + 1) { to = value; } else { clauses.add(rangeQuery(fieldName, from, to)); from = to = value; } } clauses.add(rangeQuery(fieldName, from, to)); } private void useFullText(String fieldName, Set<String> values) { if (values == null || values.isEmpty()) return; for (String value : values) { // Check for empty string final String tmp_value = StringUtils.trimLeadingCharacter(StringUtils.trimTrailingCharacter(value, '*'), '*'); if (tmp_value.isEmpty()) continue; else { final String unquoted = unquote(tmp_value); if (unquoted.isEmpty()) continue; } clauses.add(complexPhraseQuery(fieldName, StringUtils.trimLeadingCharacter(value, '*'))); } } private void useIdentifier(String fieldName, Set<String> values) { if (values == null || values.isEmpty()) return; for (String value : values) { value = StringUtils.trimTrailingCharacter(value, '*'); final String unquoted = unquote(value); if (value.isEmpty() || !unquoted.equals(value)) { clauses.add(termQuery(nonAnalyzed(fieldName), unquoted)); } else if (value.length() < IndexConstants.MIN_NGRAM_SIZE) { clauses.add(prefixQuery(fieldName, value)); } else { clauses.add(termQuery(fieldName, value)); } } } private void useIpAddressSubstring(String fieldName, Set<String> values) { if (values == null || values.isEmpty()) return; for (String value : values) { if (value.indexOf('.') >= 0) { StringBuilder sb = new StringBuilder(); sb.append("(+"); sb.append(escape(fieldName + IndexConstants.IP_ADDRESS_TYPE_SUFFIX)); sb.append(':'); sb.append(escape(IndexConstants.IP_ADDRESS_TYPE_4)); sb.append(" +"); String[] tokens = StringUtils.tokenizeToStringArray(value, "."); if (tokens.length > 0) { sb.append(ipAddressComplexPhraseQuery(fieldName, tokens)); } else { sb.append(termQuery(fieldName, value)); } sb.append(")"); clauses.add(sb.toString()); } else if (value.indexOf(':') >= 0) { StringBuilder sb = new StringBuilder(); sb.append("(+"); sb.append(escape(fieldName + IndexConstants.IP_ADDRESS_TYPE_SUFFIX)); sb.append(':'); sb.append(escape(IndexConstants.IP_ADDRESS_TYPE_6)); sb.append(" +"); String[] tokens = StringUtils.tokenizeToStringArray(value, ":"); if (tokens.length > 0) { sb.append(ipAddressComplexPhraseQuery(fieldName, tokens)); } else { sb.append(termQuery(fieldName, value)); } sb.append(")"); clauses.add(sb.toString()); } else { useWildcard(fieldName, Collections.singleton(removeLeadingZeros(value))); } } } private void useIpAddress(String fieldName, Set<InetAddress> values) { if (values == null || values.isEmpty()) return; for (InetAddress addr : values) { clauses.add(termQuery(fieldName + "_sort", IpUtils.canonicalIpAddress(addr))); } } private void useIpAddressRange(String fieldName, Set<IpRange> values) { if (values == null || values.isEmpty()) return; for (IpRange range : values) { if (range.getFrom().equals(range.getTo())) clauses.add(termQuery(fieldName + "_sort", IpUtils.canonicalIpAddress(range.getFrom()))); else clauses.add(rangeQuery(fieldName + "_sort", IpUtils.canonicalIpAddress(range.getFrom()), IpUtils.canonicalIpAddress(range.getTo()))); } } private <N extends Number & Comparable<N>> void useNumericRange(String fieldName, Set<Range<N>> ranges) { if (ranges == null || ranges.isEmpty()) return; if (occur == Occur.MUST) { for (Range<N> range : ranges) if (range.from != null || range.to != null) clauses.add(rangeQuery(fieldName, range.from, range.to)); return; } List<Range<N>> values = new ArrayList<Range<N>>(ranges); // Condense adjacent ranges Collections.sort(values); Iterator<Range<N>> it = values.iterator(); Range<N> r = it.next(); while (it.hasNext()) { Range<N> r2 = it.next(); Range<N> merged = r.merge(r2); if (merged != null) { r = merged; } else { clauses.add(rangeQuery(fieldName, r.from, r.to)); r = r2; } } clauses.add(rangeQuery(fieldName, r.from, r.to)); } private void usePath(String fieldName, Set<String> values) { if (values == null || values.isEmpty()) return; for (String value : values) { if (value.startsWith("/")) { if (value.endsWith("/")) clauses.add(prefixQuery(nonAnalyzed(fieldName), value)); else if (value.endsWith("*")) clauses.add(prefixQuery(nonAnalyzed(fieldName), StringUtils.trimTrailingCharacter(value, '*'))); else clauses.add(termQuery(nonAnalyzed(fieldName), value)); } else { clauses.add(complexPhraseQuery(fieldName, StringUtils.trimLeadingCharacter(StringUtils.trimTrailingCharacter(value, '*'), '*'))); } } } private void useTerm(String fieldName, Set<String> values) { if (values == null || values.isEmpty()) return; switch (occur) { case SHOULD: clauses.add(termsQuery(fieldName, values)); break; case MUST: case MUST_NOT: for (String value : values) clauses.add(termsQuery(fieldName, Collections.singleton(value))); break; default: throw new UnsupportedOperationException("Unexpected occur: " + occur); } } private void useWildcard(String fieldName, Set<String> values) { if (values == null || values.isEmpty()) return; for (String value : values) { // Check for empty string final String tmp_value = StringUtils.trimLeadingCharacter(StringUtils.trimTrailingCharacter(value, '*'), '*'); if (tmp_value.isEmpty()) continue; else { final String unquoted = unquote(tmp_value); if (unquoted.isEmpty()) continue; } clauses.add(escape(fieldName) + ":" + escape(value, USUAL_EXCLUDING_WILDCARDS, true)); } } private static String join(String delimiter, Collection<String> values) { if (values == null || values.isEmpty()) return null; int length = delimiter.length() * (values.size() - 1); for (String value : values) length += value.length(); StringBuilder sb = new StringBuilder(length); int i = 0; for (String value : values) { if (i++ > 0) sb.append(delimiter); sb.append(value); } return sb.toString(); } private static final char[] USUAL = "+-&|!(){}[]^\"~:\\/*?".toCharArray(); private static final char[] ESCAPE_FOR_TERMS_QUERY = "+&|!(){}[]^\"~:\\/*?,".toCharArray(); private static final char[] USUAL_EXCLUDING_WILDCARDS = "+-&|!(){}[]^\"~:\\/".toCharArray(); static { Arrays.sort(USUAL); Arrays.sort(ESCAPE_FOR_TERMS_QUERY); Arrays.sort(USUAL_EXCLUDING_WILDCARDS); } private static String escape(String s) { return escape(s, USUAL, true); } private static String escapeButDontQuote(String s) { return escape(s, USUAL, false); } private static String escape(String s, char[] escaped, boolean quote) { StringBuilder sb = new StringBuilder(); final int len = s.length(); boolean whitespace = false; for (int i = 0; i < len; i++) { final char c = s.charAt(i); if (Character.isWhitespace(c)) whitespace = true; if (Arrays.binarySearch(escaped, c) >= 0) sb.append('\\'); sb.append(c); } if (quote && (whitespace || sb.length() == 0)) return '"' + sb.append('"').toString(); else return sb.toString(); } private static String rangeQuery(final String fieldName, final Object from, final Object to) { final StringBuilder sb = new StringBuilder(); sb.append(escape(fieldName)); sb.append(":["); sb.append(from == null ? "*" : escape(from.toString())); sb.append(" TO "); sb.append(to == null ? "*" : escape(to.toString())); sb.append("]"); return sb.toString(); } private static String localParamQuery(final String query, final String... params) { StringBuilder sb = new StringBuilder("_query_:\"{!"); int i = 0; if (params.length % 2 == 1) sb.append(params[i++]); for (; i < params.length;) { if (i > 0) sb.append(" "); sb.append(escape(params[i++])); sb.append('='); sb.append(escape(params[i++])); } sb.append('}'); if (query != null) sb.append(escapeButDontQuote(query)); sb.append('"'); return sb.toString(); } private static String complexPhraseQuery(final String fieldName, final String value) { return localParamQuery(escape(fieldName) + ':' + escape(value, USUAL_EXCLUDING_WILDCARDS, true), "complexphrase"); } private static String prefixQuery(final String fieldName, final String value) { return unquote(fieldName) + ':' + escape(value) + '*'; } private static String termQuery(final String fieldName, final String value) { return unquote(fieldName) + ':' + escape(value); } private static String termsQuery(final String fieldName, final Collection<String> terms) { List<String> escapedTerms = new ArrayList<String>(terms.size()); for (String term : terms) escapedTerms.add(escape(term, ESCAPE_FOR_TERMS_QUERY, true)); return localParamQuery(join(",", escapedTerms), "terms", "f", escape(fieldName)); } private static String ipAddressComplexPhraseQuery(String fieldName, String[] tokens) { StringBuilder sb = new StringBuilder(); int i = 0; for (String token : tokens) { if (i++ > 0) sb.append(" "); sb.append(removeLeadingZeros(token)); } return complexPhraseQuery(fieldName, sb.toString()); } }