Java tutorial
/* * GeoTools - The Open Source Java GIS Toolkit * http://geotools.org * * (C) 2014, Open Source Geospatial Foundation (OSGeo) * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. */ package org.geotools.data.solr; import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import org.apache.commons.beanutils.BeanComparator; import org.apache.commons.collections.comparators.ComparatorChain; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.request.LukeRequest; import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.LukeResponse; import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; import org.apache.solr.client.solrj.response.LukeResponse.FieldTypeInfo; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.FacetParams; import org.geotools.data.Query; import org.geotools.data.solr.SolrUtils.ExtendedFieldSchemaInfo; import org.geotools.data.store.ContentDataStore; import org.geotools.data.store.ContentEntry; import org.geotools.data.store.ContentFeatureSource; import org.geotools.factory.CommonFactoryFinder; import org.geotools.factory.Hints; import org.geotools.feature.NameImpl; import org.geotools.filter.FilterCapabilities; import org.geotools.filter.visitor.SimplifyingFilterVisitor; import org.opengis.feature.simple.SimpleFeatureType; import org.opengis.feature.type.Name; import org.opengis.filter.Filter; import org.opengis.filter.FilterFactory; import org.opengis.filter.sort.SortBy; import org.opengis.filter.sort.SortOrder; import com.vividsolutions.jts.geom.Geometry; /** * Datastore implementation for SOLR document <br> * The types provided from the datastore are obtained querying with distinct a specific SOLR field */ public class SolrDataStore extends ContentDataStore { // Url of SOLR server private URL url; // Field to filter to obtain the types that the datastore provides. private String field; // Types that the datastore provides obtained by differents values of field "field" private List<Name> typeNames; // Attributes present in SOLR schema private ArrayList<SolrAttribute> solrAttributes = new ArrayList<SolrAttribute>(); // SOLR uuid attributes private SolrAttribute pk = null; // Attributes configurations of the store entries private Map<String, SolrLayerConfiguration> solrConfigurations = new ConcurrentHashMap<String, SolrLayerConfiguration>(); HttpSolrServer solrServer; /** * Create the data store * * @param url the URL of SOLR server * @param field SOLR field to query to obtain the store types */ public SolrDataStore(URL url, String field) { this.url = url; this.field = field; this.solrServer = new HttpSolrServer(url.toString()); this.solrServer.setAllowCompression(true); this.solrServer.setConnectionTimeout(10000); this.solrServer.setFollowRedirects(true); this.solrServer.setSoTimeout(10000); } /** * Retrieve SOLR attribute for specific type <br/> * Two SOLR LukeRequest are needed to discover SOLR fields and theirs schema for dynamic and * static kinds. <br/> * For each discovered field a SOLR request is needed to verify if the field has no values in * the actual type, this information will be stored in {@link SolrAttribute#setEmpty}. <br/> * SolrJ not extracts information about uniqueKey so custom class * {@link ExtendedFieldSchemaInfo} is used. <br/> * MultiValued SOLR field is mapped as String type * * @param layerName the type to use to query the SOLR field {@link SolrDataStore#field} * * @see {@link SolrUtils#decodeSolrFieldType} * @see {@link ExtendedFieldSchemaInfo#ExtendedFieldSchemaInfo} * */ public ArrayList<SolrAttribute> getSolrAttributes(String layerName) { if (solrAttributes.isEmpty()) { solrAttributes = new ArrayList<SolrAttribute>(); try { LukeRequest lq = new LukeRequest(); lq.setShowSchema(true); LukeResponse processSchema = lq.process(solrServer); lq = new LukeRequest(); lq.setShowSchema(false); LukeResponse processField = lq.process(solrServer); Map<String, FieldInfo> fis = processField.getFieldInfo(); SortedSet<String> keys = new TreeSet<String>(fis.keySet()); for (String k : keys) { FieldInfo fieldInfo = fis.get(k); String name = fieldInfo.getName(); String type = fieldInfo.getType(); SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.setRows(0); query.addFilterQuery(this.field + ":*"); if (layerName != null && layerName.isEmpty()) { query.addFilterQuery(name + ":" + layerName); } else { query.addFilterQuery(name + ":*"); } QueryResponse rsp = solrServer.query(query); long founds = rsp.getResults().getNumFound(); FieldTypeInfo fty = processSchema.getFieldTypeInfo(type); if (fty != null) { Class<?> objType = SolrUtils.decodeSolrFieldType(fty.getClassName()); if (objType != null) { ExtendedFieldSchemaInfo extendedFieldSchemaInfo = new SolrUtils.ExtendedFieldSchemaInfo( processSchema, processField, name); SolrAttribute at = new SolrAttribute(name, objType); if (extendedFieldSchemaInfo.getUniqueKey()) { at.setPk(true); at.setUse(true); } if (extendedFieldSchemaInfo.getMultivalued() && !Geometry.class.isAssignableFrom(at.getType())) { at.setType(String.class); } at.setEmpty(founds == 0); solrAttributes.add(at); } else { if (LOGGER.isLoggable(Level.FINE)) { LOGGER.log(Level.FINE, "Skipping attribute " + fty.getName() + " as we don't know how to map its type to a java object " + fty.getClassName()); } } } } // Reorder fields: empty after List<BeanComparator> sortFields = Arrays.asList(new BeanComparator("empty"), new BeanComparator("name")); ComparatorChain multiSort = new ComparatorChain(sortFields); Collections.sort(solrAttributes, multiSort); } catch (Exception ex) { LOGGER.log(Level.SEVERE, ex.getMessage(), ex); } } return solrAttributes; } @Override protected List<Name> createTypeNames() throws IOException { try { if (typeNames == null || typeNames.isEmpty()) { typeNames = new ArrayList<Name>(); SolrQuery query = new SolrQuery(); query.setQuery("*:*"); query.addFacetField(field); query.setFacet(true); query.setFacetMinCount(1); query.setFacetSort(FacetParams.FACET_SORT_INDEX); query.setRows(0); query.setParam("omitHeader", true); QueryResponse rsp = solrServer.query(query); if (LOGGER.isLoggable(Level.FINE)) { LOGGER.log(Level.FINE, "SOLR query done: " + query.toString()); } List<Count> uniqueFacetFields = rsp.getFacetFields().get(0).getValues(); for (Count field : uniqueFacetFields) { typeNames.add(new NameImpl(namespaceURI, field.getName())); } } } catch (Exception ex) { LOGGER.log(Level.SEVERE, ex.getMessage(), ex); } return typeNames; } @Override protected ContentFeatureSource createFeatureSource(ContentEntry entry) throws IOException { ContentEntry type = ensureEntry(entry.getName()); return new SolrFeatureSource(type); } @Override public FilterFactory getFilterFactory() { return CommonFactoryFinder.getFilterFactory2(); } /** * The filter capabilities which reports which spatial operations the underlying SOLR server can * handle natively. * * @return The filter capabilities, never <code>null</code>. */ public FilterCapabilities getFilterCapabilities() { FilterToSolr f2s = new FilterToSolr(); return f2s.getCapabilities(); } /** * Gets the attributes configuration for the types in this datastore */ public Map<String, SolrLayerConfiguration> getSolrConfigurations() { return solrConfigurations; } /** * Add the type configuration to this datastore */ public void setSolrConfigurations(SolrLayerConfiguration configuration) { entries.remove(new NameImpl(namespaceURI, configuration.getLayerName())); this.solrConfigurations.put(configuration.getLayerName(), configuration); } /** * Get the url of SOLR server */ public URL getUrl() { return url; } /** * Get the field used to filter the types that the datastore provides. */ public String getField() { return field; } /** * Gets the primary key attribute a type in this datastore.</br> If the key is not currently * available a call to {@link #getSolrAttributes} is needed. * * @param layerName the type to use to query the SOLR field {@link SolrDataStore#field} */ public SolrAttribute getPrimaryKey(String layerName) { if (pk == null) { ArrayList<SolrAttribute> attributes = getSolrAttributes(layerName); for (SolrAttribute at : attributes) { if (at.isPk()) { pk = at; break; } } } return pk; } /** * Builds the SolrJ query with support of subset of fields, limit/offset, sorting, OGC filter * encoding and viewParams <br> * The SOLR query always need the order by PK field to enable pagination and efficient data * retrieving <br> * Currently only additional "q" and "fq" SOLR parameters can be passed using vireParams, this * conditions are added in AND with others * * @param featureType the feature type to query * @param q the OGC query to translate in SOLR request * * @see {@link Hints#VIRTUAL_TABLE_PARAMETERS} * */ protected SolrQuery select(SimpleFeatureType featureType, Query q) { SolrQuery query = new SolrQuery(); query.setParam("omitHeader", true); try { // Column names if (q.getPropertyNames() != null) { for (String prop : q.getPropertyNames()) { query.addField(prop); } } query.setQuery("*:*"); // Encode limit/offset, if necessary if (q.getStartIndex() != null && q.getStartIndex() >= 0) { query.setStart(q.getStartIndex()); } if (q.getMaxFeatures() > 0) { query.setRows(q.getMaxFeatures()); } // Sort ORDER naturalSortOrder = ORDER.asc; if (q.getSortBy() != null) { for (SortBy sort : q.getSortBy()) { if (sort.getPropertyName() != null) { query.addSort(sort.getPropertyName().getPropertyName(), sort.getSortOrder().equals(SortOrder.ASCENDING) ? ORDER.asc : ORDER.desc); } else { naturalSortOrder = sort.getSortOrder().equals(SortOrder.ASCENDING) ? ORDER.asc : ORDER.desc; } } } // Always add natural sort by PK to support pagination query.addSort(getPrimaryKey(featureType.getTypeName()).getName(), naturalSortOrder); // Encode OGC filer FilterToSolr f2s = initializeFilterToSolr(featureType); String fq = this.field + ":" + featureType.getTypeName(); Filter simplified = SimplifyingFilterVisitor.simplify(q.getFilter()); String ffq = f2s.encodeToString(simplified); if (ffq != null && !ffq.isEmpty()) { fq = fq + " AND " + ffq; } query.setFilterQueries(fq); // Add viewpPrams addViewparams(q, query); } catch (Exception e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } return query; } /** * Builds the SolrJ count query with support of limit/offset, OGC filter encoding and viewParams <br> * Currently only additional "q" and "fq" SOLR parameters can be passed using viewParams, this * conditions are added in AND with others * * @param featureType the feature type to query * @param q the OGC query to translate in SOLR request * * @see {@link Hints#VIRTUAL_TABLE_PARAMETERS} * */ protected SolrQuery count(SimpleFeatureType featureType, Query q) { SolrQuery query = new SolrQuery(); query.setParam("omitHeader", true); query.setQuery("*:*"); query.setFields(this.getPrimaryKey(featureType.getName().getLocalPart()).getName()); try { // Encode limit/offset, if necessary if (q.getStartIndex() != null && q.getStartIndex() >= 0) { query.setStart(q.getStartIndex()); } query.setRows(0); // Encode OGC filer FilterToSolr f2s = initializeFilterToSolr(featureType); String fq = this.field + ":" + featureType.getTypeName(); String ffq = f2s.encodeToString(q.getFilter()); if (ffq != null && !ffq.isEmpty()) { fq = fq + " AND " + ffq; } query.setFilterQueries(fq); // Add viewparams parameters addViewparams(q, query); } catch (Exception e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } return query; } /* * Set parameters for OGC filter encoder */ private FilterToSolr initializeFilterToSolr(SimpleFeatureType featureType) { FilterToSolr f2s = new FilterToSolr(); f2s.setPrimaryKey(this.getPrimaryKey(featureType.getName().getLocalPart())); f2s.setFeatureTypeName(featureType.getName().getLocalPart()); return f2s; } /* * Add viewParams to SOLR query */ private void addViewparams(Query q, SolrQuery query) { String qViewParamers = null; String fqViewParamers = null; Hints hints = q.getHints(); if (hints != null) { Map<String, String> parameters = (Map<String, String>) hints.get(Hints.VIRTUAL_TABLE_PARAMETERS); if (parameters != null) { for (String param : parameters.keySet()) { // Accepts only q and fq query parameters if (param.equalsIgnoreCase("q")) { qViewParamers = parameters.get(param); } if (param.equalsIgnoreCase("fq")) { fqViewParamers = parameters.get(param); } } } } if (qViewParamers != null && !qViewParamers.isEmpty()) { query.set("q", query.get("q").concat(" AND ").concat(qViewParamers)); } if (fqViewParamers != null && !fqViewParamers.isEmpty()) { query.addFilterQuery(fqViewParamers); } } HttpSolrServer getSolrServer() { return solrServer; } @Override public void dispose() { try { solrServer.shutdown(); } finally { super.dispose(); } } }