Java tutorial
/** * Copyright 2010 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package framework.retrieval.engine.index.create.impl; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.commons.logging.Log; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import framework.retrieval.engine.analyzer.IRAnalyzerFactory; import framework.retrieval.engine.common.RetrievalUtil; import framework.retrieval.engine.context.DefaultRetrievalProperties; import framework.retrieval.engine.context.LuceneProperties; import framework.retrieval.engine.index.create.IRIndexWriter; import framework.retrieval.engine.index.create.RetrievalDocumentException; import framework.retrieval.task.ITask; import framework.retrieval.task.impl.IndexTaskMain; import framework.retrieval.task.impl.RAMIndexTask; /** * * @author * */ public class RIndexWriter implements IRIndexWriter { private Log log = RetrievalUtil.getLog(this.getClass()); private RIndexWriteProvider indexWriteProvider = null; private String indexPathType = null; private DefaultRetrievalProperties defaultRetrievalProperties = null; public RIndexWriter(IRAnalyzerFactory analyzerFactory, LuceneProperties luceneProperties, String indexPathType) { new RIndexWriter(analyzerFactory, luceneProperties, null, indexPathType); } public RIndexWriter(IRAnalyzerFactory analyzerFactory, LuceneProperties luceneProperties, DefaultRetrievalProperties defaultRetrievalProperties, String indexPathType) { this.indexPathType = indexPathType.toUpperCase(); indexWriteProvider = new RIndexWriteProvider(analyzerFactory, luceneProperties); this.defaultRetrievalProperties = defaultRetrievalProperties; } /** * ,,?? * @param documents */ public void addDocument(List<Document> documents) { if (indexPathType == null) { throw new RetrievalDocumentException("indexPathType ??null"); } if (documents == null || documents.size() <= 0) { return; } int length = documents.size(); RetrievalUtil.debugLog(log, " " + indexPathType + " " + length + " ? "); // RetrievalIndexLock.getInstance().lock(indexPathType); IndexWriter indexWriter = null; RIndexWriterWrap ramIndexWriterWrap = indexWriteProvider.createRamIndexWriter(); try { for (int i = 0; i < length; i++) { Document document = documents.get(i); ramIndexWriterWrap.getIndexWriter().addDocument(document); } ramIndexWriterWrap.getIndexWriter().commit(); indexWriter = getIndexWriter(indexPathType); indexWriter.addIndexesNoOptimize(new Directory[] { ramIndexWriterWrap.getDirectory() }); } catch (Exception e) { throw new RetrievalDocumentException(e); } finally { if (ramIndexWriterWrap != null) { try { ramIndexWriterWrap.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } if (indexWriter != null) { try { indexWriter.commit(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } try { indexWriter.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } // RetrievalIndexLock.getInstance().unlock(indexPathType); } } /** * ,,?? * ? add by sxjun * 2000?10?200?10?10?? * @param documents */ public void addDocumentNotClose(List<Document> documents, IndexWriter _indexWriter) { if (indexPathType == null) { throw new RetrievalDocumentException("indexPathType ??null"); } if (documents == null || documents.size() <= 0) { return; } int length = documents.size(); RetrievalUtil.debugLog(log, " " + indexPathType + " " + length + " ? "); int default_thread_maxnum = defaultRetrievalProperties.getDefault_thread_maxnum();// int default_single_thread_dealnums = defaultRetrievalProperties.getDefault_single_thread_dealnums();//???? int threadnum = (length % default_single_thread_dealnums > 0) ? (length / default_single_thread_dealnums + 1) : (length / default_single_thread_dealnums); if (threadnum > default_thread_maxnum) { default_single_thread_dealnums = (int) Math.ceil((double) length / default_thread_maxnum); threadnum = default_thread_maxnum; } Directory[] dirs = new Directory[threadnum]; List<ITask> tasklist = new ArrayList<ITask>(); for (int i = 0; i < threadnum; i++) { List<Document> docpage = new ArrayList<Document>(); for (int j = default_single_thread_dealnums * i; j < default_single_thread_dealnums * (i + 1) && j < length; j++) { docpage.add(documents.get(j)); } RAMIndexTask ramIndexTask = new RAMIndexTask(indexWriteProvider, docpage); tasklist.add(ramIndexTask); } List<RIndexWriterWrap> wraplist = new IndexTaskMain().exeFuture(tasklist); int size = wraplist.size(); for (int i = 0; i < size; i++) { dirs[i] = wraplist.get(i).getDirectory(); } // _indexWriter=getIndexWriter(indexPathType); try { _indexWriter.addIndexesNoOptimize(dirs); } catch (CorruptIndexException e1) { e1.printStackTrace(); } catch (IOException e1) { e1.printStackTrace(); } finally { if (wraplist != null) { for (RIndexWriterWrap w : wraplist) { if (w != null) { try { w.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } } } } } /** * , * @param documents */ public void addDocumentNowRamSupport(List<Document> documents) { if (indexPathType == null) { throw new RetrievalDocumentException("indexPathType ??null"); } if (documents == null || documents.size() <= 0) { return; } int length = documents.size(); RetrievalUtil.debugLog(log, " " + indexPathType + " " + length + " ? "); // RetrievalIndexLock.getInstance().lock(indexPathType); IndexWriter indexWriter = null; indexWriter = getIndexWriter(indexPathType); try { for (int i = 0; i < length; i++) { Document document = documents.get(i); indexWriter.addDocument(document); } } catch (Exception e) { throw new RetrievalDocumentException(e); } finally { if (indexWriter != null) { try { indexWriter.commit(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } try { indexWriter.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } // RetrievalIndexLock.getInstance().unlock(indexPathType); } } /** * * @param document */ public void addDocument(Document document) { if (indexPathType == null) { throw new RetrievalDocumentException("indexPathType ??null"); } // RetrievalIndexLock.getInstance().lock(indexPathType); IndexWriter indexWriter = null; indexWriter = getIndexWriter(indexPathType); try { RetrievalUtil.debugLog(log, " " + indexPathType + " ?"); indexWriter.addDocument(document); } catch (Exception e) { throw new RetrievalDocumentException(e); } finally { if (indexWriter != null) { try { indexWriter.commit(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } try { indexWriter.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } // RetrievalIndexLock.getInstance().unlock(indexPathType); } } /** * * @param indexPathType * @param term */ public void deleteDocument(String indexPathType, Term term) { // RetrievalIndexLock.getInstance().lock(indexPathType); IndexWriter indexWriter = null; try { RetrievalUtil.debugLog(log, "" + indexPathType); try { indexWriter = getIndexWriter(indexPathType); indexWriter.deleteDocuments(term); } catch (Exception e) { throw new RetrievalDocumentException(e); } } finally { if (indexWriter != null) { try { indexWriter.commit(); } catch (Exception e) { e.printStackTrace(); } try { indexWriter.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } // RetrievalIndexLock.getInstance().unlock(indexPathType); } } /** * ? * @param indexPathType * @param terms */ public void deleteDocument(String indexPathType, List<Term> terms) { if (terms == null || terms.size() <= 0) { return; } // RetrievalIndexLock.getInstance().lock(indexPathType); IndexWriter indexWriter = null; try { int length = terms.size(); RetrievalUtil.debugLog(log, "" + length + "" + indexPathType); try { indexWriter = getIndexWriter(indexPathType); indexWriter.deleteDocuments(terms.toArray(new Term[length])); } catch (Exception e) { e.printStackTrace(); } } finally { if (indexWriter != null) { try { indexWriter.commit(); } catch (Exception e) { e.printStackTrace(); } try { indexWriter.close(); } catch (Exception e) { RetrievalUtil.errorLog(log, e); } } // RetrievalIndexLock.getInstance().unlock(indexPathType); } } /** * ?IndexWriter * @param indexPathType * @return */ private IndexWriter getIndexWriter(String indexPathType) { IndexWriter indexWriter = indexWriteProvider.createNormalIndexWriter(indexPathType); return indexWriter; } }