Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.request.uninverted; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SegmentReader; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.*; import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.cache.Cache; import org.apache.solr.request.uninverted.GrobalCache.ILruMemSizeCache; import org.apache.solr.request.uninverted.GrobalCache.ILruMemSizeKey; import org.apache.solr.request.uninverted.UnInvertedFieldUtils.*; import com.alimama.mdrill.buffer.LuceneUtils; import com.alimama.mdrill.utils.UniqConfig; import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class UnInvertedField extends UnInvertedFieldBase { public static Logger log = LoggerFactory.getLogger(UnInvertedField.class); public static ThreadPoolExecutor SUBMIT_POOL = new ThreadPoolExecutor(UniqConfig.getUnivertedFieldThreads(), UniqConfig.getUnivertedFieldThreads(), 100L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); public static UnInvertedField getUnInvertedField(DocSet baseAdvanceDocs, String field, SolrIndexSearcher searcher) throws IOException { return getUnInvertedField(baseAdvanceDocs, field, searcher.getReader()); } public static BitDocSet ajustBase(int times, BitDocSet baseAdvanceDocs, IndexReader reader) { try { if (baseAdvanceDocs == null) { return null; } int maxdoc = reader.maxDoc(); int oversize = maxdoc / times; int size = baseAdvanceDocs.size(); int maxinterval = reader.getMaxInterval(); log.info("ajustBase " + maxinterval + ",baseAdvanceDocs=" + size + "@" + oversize + "@" + maxdoc + "," + reader.getClass().getCanonicalName()); if (size >= oversize || maxinterval > 256) { return null; } } catch (Exception e) { return null; } return baseAdvanceDocs; } public static BitDocSet cloneBitset(DocSet baseAdvanceDocs, IndexReader reader) { if (baseAdvanceDocs instanceof BitDocSet) { BitDocSet rtn = (BitDocSet) baseAdvanceDocs; OpenBitSet newbits = (OpenBitSet) (rtn.getBits().clone()); return new BitDocSet(newbits, rtn.size()); } OpenBitSet bs = new OpenBitSet(reader.maxDoc()); DocIterator iter = baseAdvanceDocs.iterator(); int pos = 0; while (iter.hasNext()) { bs.fastSet(iter.nextDoc()); pos++; } return new BitDocSet(bs, pos); } public static UnInvertedField getUnInvertedField(final DocSet baseAdvanceDocs, final String field, final SegmentReader reader, String partion, final IndexSchema schema, final boolean isreadDouble) throws IOException { final ILruMemSizeKey key = new GrobalCache.StringKey("seg@" + String.valueOf(isreadDouble) + "@" + field + "@" + reader.getStringCacheKey() + "@" + reader.getSegmentName()); ExecutorCompletionService<UnivertPool> submit = new ExecutorCompletionService<UnivertPool>(SUBMIT_POOL); final long t0 = System.currentTimeMillis(); Callable<UnivertPool> task = new Callable<UnivertPool>() { public UnivertPool call() throws Exception { UnivertPool rtnuif = new UnivertPool(); try { long t1 = System.currentTimeMillis(); Cache<ILruMemSizeKey, ILruMemSizeCache> cache = GrobalCache.fieldValueCache; final Object lockthr = UnInvertedFieldUtils.getLock(key); synchronized (lockthr) { rtnuif.uni = (UnInvertedField) cache.get(key); BitDocSet clonebitset = cloneBitset(baseAdvanceDocs, reader); long t2 = System.currentTimeMillis(); if (rtnuif.uni == null || rtnuif.uni.isShutDown()) { rtnuif.uni = new UnInvertedField(); boolean issucecess = MakeUnivertedFieldBySigment.makeInit(rtnuif.uni, clonebitset, field, reader, schema, isreadDouble); if (!issucecess) { MakeUnivertedFieldByIndex forjoin = new MakeUnivertedFieldByIndex(rtnuif.uni); forjoin.makeInit(clonebitset, field, schema, reader); } cache.put(key, rtnuif.uni); } else { boolean issucecess = MakeUnivertedFieldBySigment.addDoclist(rtnuif.uni, clonebitset, field, reader, schema, isreadDouble); if (!issucecess) { MakeUnivertedFieldByIndex forjoin = new MakeUnivertedFieldByIndex(rtnuif.uni); forjoin.addDoclist(clonebitset, field, reader); } } long t3 = System.currentTimeMillis(); log.info("####timetaken####:" + (t3 - t2) + "@" + (t2 - t1) + "@" + (t1 - t0) + "," + String.valueOf(rtnuif.uni)); } } catch (IOException e) { rtnuif.e = e; } return rtnuif; } }; submit.submit(task); UnInvertedField uif = UnInvertedFieldUtils.takeUnf(submit); uif.refCnt.incrementAndGet(); long t4 = System.currentTimeMillis(); log.info("####timetaken all####:" + (t4 - t0) + "," + String.valueOf(uif)); return uif; } public static UnInvertedField getUnInvertedField(final DocSet baseAdvanceDocs, final String field, SolrIndexReader reader) throws IOException { final SolrIndexSearcher searcher = reader.getSearcher(); final ILruMemSizeKey key = new GrobalCache.StringKey( searcher.getPartionKey() + "@@" + field + "@@" + LuceneUtils.crcKey(reader)); final long t0 = System.currentTimeMillis(); ExecutorCompletionService<UnivertPool> submit = new ExecutorCompletionService<UnivertPool>(SUBMIT_POOL); Callable<UnivertPool> task = new Callable<UnivertPool>() { public UnivertPool call() throws Exception { UnivertPool rtnuif = new UnivertPool(); try { final Cache<ILruMemSizeKey, ILruMemSizeCache> cache = GrobalCache.fieldValueCache; final Object lockthr = UnInvertedFieldUtils.getLock(key); synchronized (lockthr) { long t1 = System.currentTimeMillis(); rtnuif.uni = (UnInvertedField) cache.get(key); SolrIndexReader reader = searcher.getReader(); BitDocSet clonebitset = cloneBitset(baseAdvanceDocs, reader); if (rtnuif.uni == null || rtnuif.uni.isShutDown()) { rtnuif.uni = new UnInvertedField(); MakeUnivertedFieldByIndex forjoin = new MakeUnivertedFieldByIndex(rtnuif.uni); forjoin.makeInit(clonebitset, field, searcher.getSchema(), reader); cache.put(key, rtnuif.uni); } else { MakeUnivertedFieldByIndex forjoin = new MakeUnivertedFieldByIndex(rtnuif.uni); forjoin.addDoclist(clonebitset, field, reader); } long t2 = System.currentTimeMillis(); log.info("timetaken by index:" + (t2 - t1) + "@" + (t1 - t0) + "," + String.valueOf(rtnuif.uni)); } } catch (IOException e) { rtnuif.e = e; } return rtnuif; } }; submit.submit(task); UnInvertedField uif = UnInvertedFieldUtils.takeUnf(submit); uif.refCnt.incrementAndGet(); return uif; } }