List of usage examples for org.apache.solr.core SolrResourceLoader getLines
public List<String> getLines(String resource) throws IOException
From source file:de.dlr.knowledgefinder.dataimport.utils.transformer.ExcludeValuesTransformer.java
License:Apache License
private CharArraySet initializeExcludeValues(String filename) throws IOException { CharArraySet loadedFile = loadedFiles.get(filename); if (loadedFile == null) { SolrCore core = context.getSolrCore(); SolrResourceLoader loader = core.getResourceLoader(); List<String> lines = Collections.emptyList(); lines = loader.getLines(filename); loadedFiles.put(filename, new CharArraySet(lines, true)); }/* ww w.j a v a2 s.c o m*/ return loadedFiles.get(filename); }
From source file:NomusSolrPlugins.NomusDismaxQParserPlugin.java
License:Apache License
public Query parse() throws ParseException { SolrParams localParams = getLocalParams(); SolrParams params = getParams();// w w w .ja v a2s . c o m SolrParams solrParams = localParams == null ? params : new DefaultSolrParams(localParams, params); // load the field name synonyms HashMap<String, String> fieldSynonyms = new HashMap(); try { SolrResourceLoader loader = req.getCore().getResourceLoader(); List<String> lines = loader.getLines("field-synonyms.txt"); for (String line : lines) { String[] fieldSynStrs = line.split(":"); for (int i = 1; i < fieldSynStrs.length; i++) { fieldSynonyms.put(fieldSynStrs[0], fieldSynStrs[i]); } } } catch (java.io.IOException e) { throw new ParseException(e.toString()); } queryFields = U.parseFieldBoosts(solrParams.getParams(DMP.QF)); applyFieldSynonyms(fieldSynonyms, queryFields); /*if (0 == queryFields.size()) { queryFields.put(req.getSchema().getDefaultSearchFieldName(), 1.0f); }*/ // // Query for which the query is run only to boost // matches of the main results Map<String, Float> optionalFields = U.parseFieldBoosts(solrParams.getParams("of")); applyFieldSynonyms(fieldSynonyms, optionalFields); // Boosted phrase of the full query string Map<String, Float> phraseFields = U.parseFieldBoosts(solrParams.getParams(DMP.PF)); applyFieldSynonyms(fieldSynonyms, phraseFields); // Boosted Bi-Term Shingles from the query string Map<String, Float> phraseFields2 = U.parseFieldBoosts(solrParams.getParams("pf2")); applyFieldSynonyms(fieldSynonyms, phraseFields2); // Boosted Tri-Term Shingles from the query string Map<String, Float> phraseFields3 = U.parseFieldBoosts(solrParams.getParams("pf3")); applyFieldSynonyms(fieldSynonyms, phraseFields3); float tiebreaker = solrParams.getFloat(DMP.TIE, 0.0f); int pslop = solrParams.getInt(DMP.PS, 0); int qslop = solrParams.getInt(DMP.QS, 0); // remove stopwords from mandatory "matching" component? boolean stopwords = solrParams.getBool("stopwords", true); /* the main query we will execute. we disable the coord because * this query is an artificial construct */ BooleanQuery query = new BooleanQuery(true); /* * * Main User Query * * */ parsedUserQuery = null; String userQuery = getString(); altUserQuery = null; if (userQuery == null || userQuery.length() < 1) { // If no query is specified, we may have an alternate String altQ = solrParams.get(DMP.ALTQ); if (altQ != null) { altQParser = subQuery(altQ, null); altUserQuery = altQParser.getQuery(); query.add(altUserQuery, BooleanClause.Occur.MUST); } else { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing query string"); } } else { // There is a valid query string // userQuery = partialEscape(U.stripUnbalancedQuotes(userQuery)).toString(); boolean lowercaseOperators = solrParams.getBool("lowercaseOperators", true); String mainUserQuery = userQuery; // User query parser ExtendedSolrQueryParser up = new ExtendedSolrQueryParser(this, IMPOSSIBLE_FIELD_NAME); up.addAlias(IMPOSSIBLE_FIELD_NAME, tiebreaker, queryFields); up.setPhraseSlop(qslop); // slop for explicit user phrase queries up.setAllowLeadingWildcard(true); // defer escaping and only do if lucene parsing fails, or we need phrases // parsing fails. Need to sloppy phrase queries anyway though. List<Clause> clauses = null; boolean specialSyntax = false; int numPluses = 0; int numMinuses = 0; int numOptional = 0; int numAND = 0; int numOR = 0; int numNOT = 0; boolean sawLowerAnd = false; boolean sawLowerOr = false; boolean sawAmpersand = false; clauses = splitIntoClauses(userQuery, false); for (Clause clause : clauses) { if (!clause.isPhrase && clause.hasSpecialSyntax) { specialSyntax = true; } if (clause.must == '+') numPluses++; if (clause.must == '-') numMinuses++; if (clause.isBareWord()) { String s = clause.val; if ("AND".equals(s)) { numAND++; } else if ("OR".equals(s)) { numOR++; } else if ("NOT".equals(s)) { numNOT++; } else if ("&".equals(s)) { numAND++; sawAmpersand = true; } else if (lowercaseOperators) { if ("and".equals(s)) { numAND++; sawLowerAnd = true; } else if ("or".equals(s)) { numOR++; sawLowerOr = true; } } } } numOptional = clauses.size() - (numPluses + numMinuses); // convert lower or mixed case operators to uppercase if we saw them. // only do this for the lucene query part and not for phrase query boosting // since some fields might not be case insensitive. // We don't use a regex for this because it might change and AND or OR in // a phrase query in a case sensitive field. // also change "&" to AND if (sawLowerAnd || sawLowerOr || sawAmpersand) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < clauses.size(); i++) { Clause clause = clauses.get(i); String s = clause.raw; // and and or won't be operators at the start or end if (i > 0 && i + 1 < clauses.size()) { if ("AND".equalsIgnoreCase(s)) { s = "AND"; } else if ("OR".equalsIgnoreCase(s)) { s = "OR"; } else if ("&".equals(s)) { s = "AND"; } } sb.append(s); sb.append(' '); } mainUserQuery = sb.toString(); } // For correct lucene queries, turn off mm processing if there // were explicit operators (except for AND). boolean doMinMatched = (numOR + numNOT + numPluses + numMinuses) == 0; try { up.setRemoveStopFilter(!stopwords); parsedUserQuery = up.parse(mainUserQuery); if (stopwords && isEmpty(parsedUserQuery)) { // if the query was all stop words, remove none of them up.setRemoveStopFilter(true); parsedUserQuery = up.parse(mainUserQuery); } } catch (Exception e) { // ignore failure and reparse later after escaping reserved chars } if (parsedUserQuery != null && doMinMatched) { String minShouldMatch = solrParams.get(DMP.MM, "100%"); if (parsedUserQuery instanceof BooleanQuery) { U.setMinShouldMatch((BooleanQuery) parsedUserQuery, minShouldMatch); } } String escapedUserQuery = null; if (parsedUserQuery == null) { StringBuilder sb = new StringBuilder(); for (Clause clause : clauses) { boolean doQuote = clause.isPhrase; String s = clause.val; if (!clause.isPhrase && ("OR".equals(s) || "AND".equals(s) || "NOT".equals(s))) { doQuote = true; } if (clause.must != 0) { sb.append(clause.must); } if (clause.field != null) { sb.append(clause.field); sb.append(':'); } if (doQuote) { sb.append('"'); } sb.append(clause.val); if (doQuote) { sb.append('"'); } sb.append(' '); } escapedUserQuery = sb.toString(); Query escapedParsedUserQuery = up.parse(escapedUserQuery); // Only do minimum-match logic String minShouldMatch = solrParams.get(DMP.MM, "100%"); if (escapedParsedUserQuery instanceof BooleanQuery) { BooleanQuery t = new BooleanQuery(); U.flattenBooleanQuery(t, (BooleanQuery) escapedParsedUserQuery); U.setMinShouldMatch(t, minShouldMatch); escapedParsedUserQuery = t; } // use the escaped query if (0 != queryFields.size()) { query.add(escapedParsedUserQuery, BooleanClause.Occur.MUST); } } else { // no need to use escaped query - go with parsed if (0 != queryFields.size()) { query.add(parsedUserQuery, BooleanClause.Occur.MUST); } } // re-use the parser on the optional fields up.clearAliases(); up.addAlias(IMPOSSIBLE_FIELD_NAME, tiebreaker, optionalFields); Query optionalQuery = null; if (parsedUserQuery == null) optionalQuery = up.parse(escapedUserQuery); else optionalQuery = up.parse(mainUserQuery); query.add(optionalQuery, BooleanClause.Occur.SHOULD); // sloppy phrase queries for proximity if (phraseFields.size() > 0 || phraseFields2.size() > 0 || phraseFields3.size() > 0) { // find non-field clauses List<Clause> normalClauses = new ArrayList<Clause>(clauses.size()); for (Clause clause : clauses) { if (clause.field != null || clause.isPhrase) continue; // check for keywords "AND,OR,TO" if (clause.isBareWord()) { String s = clause.val.toString(); // avoid putting explict operators in the phrase query if ("OR".equals(s) || "AND".equals(s) || "NOT".equals(s) || "TO".equals(s)) continue; } normalClauses.add(clause); } // full phrase... addShingledPhraseQueries(query, normalClauses, phraseFields, 0, tiebreaker, pslop); // shingles... addShingledPhraseQueries(query, normalClauses, phraseFields2, 2, tiebreaker, pslop); addShingledPhraseQueries(query, normalClauses, phraseFields3, 3, tiebreaker, pslop); } } /* * * Boosting Query * * */ boostParams = solrParams.getParams(DMP.BQ); //List<Query> boostQueries = U.parseQueryStrings(req, boostParams); boostQueries = null; if (boostParams != null && boostParams.length > 0) { boostQueries = new ArrayList<Query>(); for (String qs : boostParams) { if (qs.trim().length() == 0) continue; Query q = subQuery(qs, null).getQuery(); boostQueries.add(q); } } if (null != boostQueries) { for (Query f : boostQueries) { query.add(f, BooleanClause.Occur.SHOULD); } } /* * * Boosting Functions * * */ String[] boostFuncs = solrParams.getParams(DMP.BF); if (null != boostFuncs && 0 != boostFuncs.length) { for (String boostFunc : boostFuncs) { if (null == boostFunc || "".equals(boostFunc)) continue; Map<String, Float> ff = SolrPluginUtils.parseFieldBoosts(boostFunc); for (String f : ff.keySet()) { Query fq = subQuery(f, FunctionQParserPlugin.NAME).getQuery(); Float b = ff.get(f); if (null != b) { fq.setBoost(b); } query.add(fq, BooleanClause.Occur.SHOULD); } } } // // create a boosted query (scores multiplied by boosts) // Query topQuery = query; multBoosts = solrParams.getParams("boost"); if (multBoosts != null && multBoosts.length > 0) { List<ValueSource> boosts = new ArrayList<ValueSource>(); for (String boostStr : multBoosts) { if (boostStr == null || boostStr.length() == 0) continue; Query boost = subQuery(boostStr, FunctionQParserPlugin.NAME).getQuery(); ValueSource vs; if (boost instanceof FunctionQuery) { vs = ((FunctionQuery) boost).getValueSource(); } else { vs = new QueryValueSource(boost, 1.0f); } boosts.add(vs); } if (boosts.size() > 1) { ValueSource prod = new ProductFloatFunction(boosts.toArray(new ValueSource[boosts.size()])); topQuery = new BoostedQuery(query, prod); } else if (boosts.size() == 1) { topQuery = new BoostedQuery(query, boosts.get(0)); } } return topQuery; }