Java tutorial
/* * Copyright 2008-2011 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.nominanuda.roafeed; import java.io.IOException; import java.util.Collection; import java.util.LinkedList; import java.util.Set; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrInputDocument; import com.nominanuda.codec.Digester; import com.nominanuda.dataobject.DataArray; import com.nominanuda.dataobject.DataObject; import com.nominanuda.dataobject.DataObjectImpl; import com.nominanuda.dataobject.DataStructHelper; import com.nominanuda.dataobject.DataStructSelector; import com.nominanuda.dataobject.DomainObjectHelper; import com.nominanuda.dataobject.Struct; import com.nominanuda.lang.Check; import com.nominanuda.lang.Collections; import com.nominanuda.lang.DateTimeHelper; import com.nominanuda.roafeed.FeedHyperApi.Bucket; import com.nominanuda.roafeed.FeedHyperApi.Entry; import com.nominanuda.roafeed.FeedHyperApi.FeedDownloadTask; import com.nominanuda.roafeed.FeedHyperApi.Job; import com.nominanuda.solr.SolrHelper; import com.nominanuda.web.http.HttpProtocol; public class SolrFeedDao implements FeedDao { //private static final Logger log = LoggerFactory.getLogger(SolrFeedDao.class); private static final DataStructSelector selector = new DataStructSelector(); private static final DataStructHelper struct = new DataStructHelper(); private static final DateTimeHelper dt = new DateTimeHelper(); private final Digester digester = new Digester().withCharset("UTF-8"); private SolrServer solr; private SolrHelper s = new SolrHelper(); //TODO private Set<String> solrFields = Collections.hashSet("id", "type_", "feedType", "body", "feedUrl", "userData.userTitle", "url", "bucket", "lastRun", "lastSuccess", "cron", "nextRun", "wfStatus", "job"); private DomainObjectHelper domain = new DomainObjectHelper(); private String t_bucket = domain.domainTypeName(Bucket.class); private String t_job = domain.domainTypeName(Job.class); private String t_feeddownloadtask = domain.domainTypeName(FeedDownloadTask.class); private String t_fingerprint = "fingerprint"; private String t_entry = domain.domainTypeName(Entry.class); public void createOrUpdateBucket(DataObject bucketData) throws SolrServerException, IOException { bucketData.put("id", t_bucket + "_" + bucketData.getStrict("bucket").toString()); bucketData.put("type_", t_bucket); solr.add(s.sid(bucketData, solrFields), 0); solr.commit(true, true); } public DataObject getBucketByName(String bucket) throws SolrServerException { SolrQuery sq = new SolrQuery("id:" + t_bucket + "_" + bucket); DataArray res = s.listResultsDotAware(solr, sq, 0, 1); return res.getLength() > 0 ? res.getObject(0) : null; } public Iterable<DataObject> findNextFeedsToUpdate(int count) throws SolrServerException { SolrQuery sq = new SolrQuery( "type_:" + t_feeddownloadtask + " AND nextRun:[* TO " + dt.nowToISO8601UtcSecs() + "]"); sq.setSortField("nextRun", ORDER.desc); DataArray res = s.listResultsDotAware(solr, sq, 0, count); return struct.castAsIterable(res); } public void updateFeedDownloadTask(DataObject feedTask) throws SolrServerException, IOException { solr.add(s.sid(feedTask, solrFields), 0); solr.commit(true, true); } public boolean existsEntryFingerprint(String fingerprint) throws SolrServerException { SolrQuery sq = new SolrQuery("id:fingerprint_" + fingerprint); DataArray res = s.listResultsDotAware(solr, sq, 0, 1); return res.getLength() > 0; } public void storeFingerprint(String fingerprint) throws SolrServerException, IOException { DataObject o = new DataObjectImpl(); o.put("type_", t_fingerprint); o.put("id", t_fingerprint + "_" + fingerprint); solr.add(s.sid(o, solrFields), 0); //solr.commit(true, true); } public void addEntryToQueue(String entry, String feedUrl, String feedType/*, DataArray buckets*/, @Struct(cls = FeedDownloadTask.class) DataObject downloadTask) throws SolrServerException, IOException { DataArray jobs = findJobsByFeedUrl(feedUrl); DataObject o = new DataObjectImpl(); String entryHash = digester.sha1(entry).toBase62(); o.put("type_", t_entry); //o.put("id", t_entry+":"+entryHash); o.put("feedType", feedType); o.put("feedUrl", feedUrl); o.put("body", entry); Collection<SolrInputDocument> docs = new LinkedList<SolrInputDocument>(); for (Object _j : jobs) { DataObject _job = (DataObject) _j; String bucketName = _job.getStrict("bucket").toString(); DataObject o1 = struct.clone(o); o1.put("bucket", bucketName); o1.put("id", t_entry + "_" + bucketName + '_' + entryHash); if (_job.exists("userData")) { o1.put("userData", _job.get("userData")); } else { DataObject bucket = getBucketByName(bucketName); if (bucket.exists("userData")) { o1.put("userData", bucket.get("userData")); } } docs.add(s.sid(o1, solrFields)); } solr.add(docs, 0); solr.commit(true, true); } private DataArray findJobsByFeedUrl(String feedUrl) throws SolrServerException { SolrQuery sq = new SolrQuery("type_:" + t_job + " AND url:" + s.escAndQuote(feedUrl)); DataArray res = s.listResultsDotAware(solr, sq, 0, 1); return res; } // private @Nullable DataObject findJobByBucketAndUrl(String bucket, String url) throws SolrServerException { // SolrQuery sq = new SolrQuery("type_:feedjob AND bucket:"+s.escAndQuote(bucket)+" AND url:"+s.escAndQuote(url)); // DataArray res = s.listResultsDotAware(solr, sq, 0, 1); // return res.getLength() == 0 ? null : res.getObject(0); // } public DataArray listBucket(String bucket, int start, int count) throws SolrServerException { SolrQuery sq = new SolrQuery("type_:" + t_job + " AND bucket:" + s.escAndQuote(bucket)); sq.setSortField("job", ORDER.asc); DataArray res = s.listResultsDotAware(solr, sq, start, count); return res; } public void removeEntry(String id) throws SolrServerException, IOException { Check.illegalargument.assertTrue(id.startsWith(t_entry)); solr.deleteById(id); solr.commit(true, true); } public void createOrUpdateJob(DataObject jobData) throws SolrServerException, IOException { final String _bucket = jobData.getStrict("bucket").toString(); final String job = jobData.getStrict("job").toString(); jobData.put("id", "job_" + _bucket + "_" + job); jobData.put("type_", "job"); solr.add(s.sid(jobData, solrFields), 0); String url = jobData.getStrict("url").toString(); DataObject fdt = getFeedDownloadTaskByUrl(url); if (fdt == null) { fdt = new DataObjectImpl(); fdt.put("type_", "feeddownloadtask"); fdt.put("url", url); fdt.put("feedType", "atom_1.0"); fdt.putNewArray("bucket").add(_bucket); fdt.put("job", job); fdt.put("id", "feeddownloadtask_" + digester.sha1(url.getBytes(HttpProtocol.UTF_8)).toBase62()); fdt.put("wfStatus", 2); } else { Object buckets = fdt.get("bucket"); if (buckets instanceof DataArray) { ((DataArray) buckets).add(_bucket); } else { DataArray a = fdt.putNewArray("bucket"); a.add(buckets); a.add(_bucket); } } fdt.put("nextRun", dt.nowToISO8601UtcSecs()); solr.add(s.sid(fdt, solrFields), 0); solr.commit(true, true); } private DataObject getFeedDownloadTaskByUrl(String url) throws SolrServerException { SolrQuery sq = new SolrQuery("type_:feeddownloadtask AND url:" + s.escAndQuote(url) + " "); DataArray res = s.listResultsDotAware(solr, sq, 0, 1); return res.getLength() == 0 ? null : res.getObject(0); } public Iterable<DataObject> getNextEntriesByBucket(String bucket, int count) throws SolrServerException { SolrQuery sq = new SolrQuery("type_:entry AND bucket:" + s.escAndQuote(bucket) + " "); DataArray res = s.listResultsDotAware(solr, sq, 0, count); return struct.castAsIterable(res); } public DataObject getJobByBucketAndName(String bucket, String job) throws SolrServerException { SolrQuery sq = new SolrQuery("id:" + s.escAndQuote("job_" + bucket + "_" + job)); DataArray res = s.listResultsDotAware(solr, sq, 0, 1); return res.getLength() == 0 ? null : res.getObject(0); } public Iterable<String> listBuckets(int start, int count) throws SolrServerException { SolrQuery sq = new SolrQuery("type_:bucket"); sq.setSortField("type_", ORDER.asc); DataArray res = s.listResultsDotAware(solr, sq, start, count); return struct.castAsIterable(selector.select(res, "bucket", DataArray.class)); } public void setSolr(SolrServer solr) { this.solr = solr; } void TODO() { //SOLR doc patch } }