Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.contrib.index.mapred; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.Collection; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.hadoop.contrib.index.lucene.RAMDirectoryUtil; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; /** * An intermediate form for one or more parsed Lucene documents and/or * delete terms. It actually uses Lucene file format as the format for * the intermediate form by using RAM dir files. * * Note: If process(*) is ever called, closeWriter() should be called. * Otherwise, no need to call closeWriter(). */ public class IntermediateForm implements Writable { private IndexUpdateConfiguration iconf = null; private final Collection<Term> deleteList; private RAMDirectory dir; private IndexWriter writer; private int numDocs; /** * Constructor * @throws IOException */ public IntermediateForm() throws IOException { deleteList = new ConcurrentLinkedQueue<Term>(); dir = new RAMDirectory(); writer = null; numDocs = 0; } /** * Configure using an index update configuration. * @param iconf the index update configuration */ public void configure(IndexUpdateConfiguration iconf) { this.iconf = iconf; } /** * Get the ram directory of the intermediate form. * @return the ram directory */ public Directory getDirectory() { return dir; } /** * Get an iterator for the delete terms in the intermediate form. * @return an iterator for the delete terms */ public Iterator<Term> deleteTermIterator() { return deleteList.iterator(); } /** * This method is used by the index update mapper and process a document * operation into the current intermediate form. * @param doc input document operation * @param analyzer the analyzer * @throws IOException */ public void process(DocumentAndOp doc, Analyzer analyzer) throws IOException { if (doc.getOp() == DocumentAndOp.Op.DELETE || doc.getOp() == DocumentAndOp.Op.UPDATE) { deleteList.add(doc.getTerm()); } if (doc.getOp() == DocumentAndOp.Op.INSERT || doc.getOp() == DocumentAndOp.Op.UPDATE) { if (writer == null) { // analyzer is null because we specify an analyzer with addDocument writer = createWriter(); } writer.addDocument(doc.getDocument(), analyzer); numDocs++; } } /** * This method is used by the index update combiner and process an * intermediate form into the current intermediate form. More specifically, * the input intermediate forms are a single-document ram index and/or a * single delete term. * @param form the input intermediate form * @throws IOException */ public void process(IntermediateForm form) throws IOException { if (form.deleteList.size() > 0) { deleteList.addAll(form.deleteList); } if (form.dir.sizeInBytes() > 0) { if (writer == null) { writer = createWriter(); } writer.addIndexesNoOptimize(new Directory[] { form.dir }); numDocs++; } } /** * Close the Lucene index writer associated with the intermediate form, * if created. Do not close the ram directory. In fact, there is no need * to close a ram directory. * @throws IOException */ public void closeWriter() throws IOException { if (writer != null) { writer.close(); writer = null; } } /* (non-Javadoc) * @see java.lang.Object#toString() */ public String toString() { StringBuilder buffer = new StringBuilder(); buffer.append(this.getClass().getSimpleName()); buffer.append("[numDocs="); buffer.append(numDocs); buffer.append(", numDeletes="); buffer.append(deleteList.size()); if (deleteList.size() > 0) { buffer.append("("); Iterator<Term> iter = deleteTermIterator(); while (iter.hasNext()) { buffer.append(iter.next()); buffer.append(" "); } buffer.append(")"); } buffer.append("]"); return buffer.toString(); } private IndexWriter createWriter() throws IOException { IndexWriter writer = new IndexWriter(dir, false, null, new KeepOnlyLastCommitDeletionPolicy()); writer.setUseCompoundFile(false); if (iconf != null) { int maxFieldLength = iconf.getIndexMaxFieldLength(); if (maxFieldLength > 0) { writer.setMaxFieldLength(maxFieldLength); } } return writer; } private void resetForm() throws IOException { deleteList.clear(); if (dir.sizeInBytes() > 0) { // it's ok if we don't close a ram directory dir.close(); // an alternative is to delete all the files and reuse the ram directory dir = new RAMDirectory(); } assert (writer == null); numDocs = 0; } // /////////////////////////////////// // Writable // /////////////////////////////////// /* (non-Javadoc) * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput) */ public void write(DataOutput out) throws IOException { out.writeInt(deleteList.size()); for (Term term : deleteList) { Text.writeString(out, term.field()); Text.writeString(out, term.text()); } String[] files = dir.list(); RAMDirectoryUtil.writeRAMFiles(out, dir, files); } /* (non-Javadoc) * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput) */ public void readFields(DataInput in) throws IOException { resetForm(); int numDeleteTerms = in.readInt(); for (int i = 0; i < numDeleteTerms; i++) { String field = Text.readString(in); String text = Text.readString(in); deleteList.add(new Term(field, text)); } RAMDirectoryUtil.readRAMFiles(in, dir); } }