Java tutorial
/********************************************************************************** * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/index/impl/ClusterFSIndexStorage.java $ * $Id: ClusterFSIndexStorage.java 111640 2012-08-20 12:58:11Z david.horwitz@uct.ac.za $ *********************************************************************************** * * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008, 2009 The Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.opensource.org/licenses/ECL-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * **********************************************************************************/ package org.sakaiproject.search.index.impl; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.MultiReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.sakaiproject.search.index.ClusterFilesystem; import org.sakaiproject.search.index.SegmentInfo; /** * Implementation of IndexStorage using a Cluster File system. This * implementation performs all index write operations in a new temporary segment. * On completion of the index operation it is merged with the current segment. * If the current segment is larger than the threshold, a new segment is * created. Managing the segments and how they relate to the cluster is * delegated to the ClusterFilesystem * * @author ieb */ public class ClusterFSIndexStorage extends BaseIndexStorage { private static final Log log = LogFactory.getLog(ClusterFSIndexStorage.class); /** * Maximum size of a segment on write */ private long segmentThreshold = 1024 * 1024 * 20; // Maximum Segment size // is 20M private ClusterFilesystem clusterFS = null; // maximum size of a segment during merge private long maxSegmentSize = 1024L * 1024L * 1500L; // just short of // 1.5G // maximum size of a segment considered for merge operations private long maxMegeSegmentSize = 1024L * 1024L * 1200L; // 1.2G public void init() { } public IndexReader getIndexReader() throws IOException { return getIndexReader(true); } private IndexReader getIndexReader(boolean withLock) throws IOException { if (withLock) { clusterFS.getLock(); } List<SegmentInfo> segments = clusterFS.updateSegments(); if (log.isDebugEnabled()) log.debug("Found " + segments.size() + " segments "); IndexReader[] readers = new IndexReader[segments.size()]; int j = 0; for (Iterator<SegmentInfo> i = segments.iterator(); i.hasNext();) { SegmentInfo segment = i.next(); try { if (!segment.checkSegmentValidity(diagnostics, "getIndexReader ")) { log.warn("Checksum Failed on " + segment); segment.checkSegmentValidity(true, "getIndexReader Failed"); } readers[j] = IndexReader.open(FSDirectory.open(segment.getSegmentLocation()), false); } catch (Exception ex) { try { if (readers[j] != null) { try { readers[j].close(); readers[j] = null; } catch (Exception e) { log.debug(e); } } if (log.isDebugEnabled()) log.debug("Invalid segment ", ex); log.warn("Found corrupted segment (" + segment.getName() + ") in Local store, attempting to recover from DB. Reason: " + ex.getClass().getName() + ":" + ex.getMessage(), ex); clusterFS.recoverSegment(segment); readers[j] = IndexReader.open(FSDirectory.open(segment.getSegmentLocation()), false); log.warn( "Recovery complete, resuming normal operations having restored, ignore previous problems with this segment " + segment.getName()); } catch (Exception e) { if (readers[j] != null) { try { readers[j].close(); readers[j] = null; } catch (Exception ex2) { log.debug(e); } } log.error("---Problem recovering corrupted segment from the DB,\n" + "--- it is probably that there has been a local hardware\n" + "--- failure on this node or that the backup in the DB is missing\n" + "--- or corrupt. To recover, remove the segment from the db, and rebuild the index \n" + "--- eg delete from search_segments where name_ = '" + segment.getName() + "'; \n", ex); } } j++; } List<IndexReader> l = new ArrayList<IndexReader>(); for (int i = 0; i < readers.length; i++) { if (readers[i] != null) { l.add(readers[i]); } } if (l.size() != readers.length) { log.warn( " Opening index reader with a partial index set, this may result in a smallere search set than otherwise expected"); } readers = l.toArray(new IndexReader[0]); if (readers.length > 0) { IndexReader indexReader = new MultiReader(readers); return indexReader; } throw new IOException("No Index available to open "); } public IndexWriter getIndexWriter(boolean create) throws IOException { if (log.isDebugEnabled()) log.debug("+++++++++++++++++Start Index Writer Cycle "); // to ensure that we dont dammage the index due to OutOfMemory, if it // should ever happen // we will open a temporary index, which will be merged on completion IndexWriter indexWriter = null; File tempIndex = clusterFS.getTemporarySegment(true); indexWriter = new IndexWriter(FSDirectory.open(tempIndex), getAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter.setUseCompoundFile(true); // indexWriter.setInfoStream(System.out); indexWriter.setMaxMergeDocs(50); indexWriter.setMergeFactor(50); if (log.isDebugEnabled()) log.debug("Using Temp Index Writer " + tempIndex.getPath()); return indexWriter; } protected IndexSearcher getIndexSearcher() throws IOException { IndexSearcher indexSearcher = null; try { long reloadStart = System.currentTimeMillis(); log.debug("Open Search"); indexSearcher = new IndexSearcher(getIndexReader(false)); if (indexSearcher == null) { log.warn("No search Index exists at this time"); } long reloadEnd = System.currentTimeMillis(); if (log.isDebugEnabled()) log.debug("Reload Complete " + indexSearcher.maxDoc() + " in " + (reloadEnd - reloadStart)); } catch (FileNotFoundException e) { try { if (indexSearcher != null) { indexSearcher.close(); } } catch (Exception ex) { log.debug(e); } indexSearcher = null; log.error("There has been a major poblem with the" + " Search Index which has become corrupted ", e); } catch (IOException e) { try { if (indexSearcher != null) { indexSearcher.close(); } } catch (Exception ex) { log.debug(e); } indexSearcher = null; log.error("There has been a major poblem with the " + "Search Index which has become corrupted", e); } return indexSearcher; } public boolean indexExists() { List<SegmentInfo> segments = clusterFS.updateSegments(); return (segments.size() > 0); } public void doPreIndexUpdate() throws IOException { if (log.isDebugEnabled()) log.debug("Start Index Cycle"); // dont enable locks //FSDirectory.setDisableLocks(true); } public void doPostIndexUpdate() throws IOException { } private void mergeAndUpdate(boolean merge) throws IOException { if (merge) { //FSDirectory.setDisableLocks(true); // get the tmp index File tmpSegment = clusterFS.getTemporarySegment(false); Directory[] tmpDirectory = new Directory[1]; tmpDirectory[0] = FSDirectory.open(tmpSegment); // Need to fix checksums before merging.... is that really true, // List<SegmentInfo> segments = clusterFS.updateSegments(); if (log.isDebugEnabled()) log.debug("Merge Phase 1: Starting on " + segments.size() + " segments "); // merge it with the current index SegmentInfo currentSegment = null; if (log.isDebugEnabled()) log.debug("Found " + segments.size() + " segments "); if (segments.size() > 0) { currentSegment = segments.get(segments.size() - 1); if (currentSegment != null) { if (!currentSegment.isClusterSegment() || (currentSegment.getTotalSize() > segmentThreshold) || currentSegment.isDeleted()) { if (diagnostics) { log.info("Curre nt Segment not suitable, generating new segment " + (currentSegment.isDeleted() ? "deleted," : "") + (!currentSegment.isClusterSegment() ? "non-cluster," : "") + ((currentSegment.getTotalSize() > segmentThreshold) ? "toobig," : "")); } currentSegment = null; } } } if (currentSegment == null) { if (tmpDirectory[0].fileExists("segments.gen")) { currentSegment = clusterFS.saveTemporarySegment(); /* * We must add the new current segment to the list of * segments so if it gets merged in the next step is is not * left out */ segments.add(currentSegment); /* * We should touch the segment to notify that it has been * updated */ currentSegment.touchSegment(); } else { log.warn( "No Segment Created during indexing process, this should not happen, although it is possible tha the indexing operation did not find any files to index."); } } else { IndexWriter indexWriter = null; try { if (log.isDebugEnabled()) log.debug("Using Existing Segment " + currentSegment.getName()); currentSegment.touchSegment(); Directory dir = FSDirectory.open(currentSegment.getSegmentLocation()); indexWriter = new IndexWriter(dir, getAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter.setUseCompoundFile(true); // indexWriter.setInfoStream(System.out); indexWriter.setMaxMergeDocs(50); indexWriter.setMergeFactor(50); if (tmpDirectory[0].fileExists("segments.gen")) { if (log.isDebugEnabled()) log.debug("Merging Temp segment " + tmpSegment.getPath() + " with current segment " + currentSegment.getSegmentLocation().getPath()); indexWriter.addIndexesNoOptimize(tmpDirectory); indexWriter.optimize(); } else { log.warn("No Merge performed, no tmp segment"); } } finally { try { indexWriter.close(); currentSegment.touchSegment(); } catch (Exception ex) { // dotn care if this fails log.debug(ex); } } } /* * segments in now a list of all segments including the current * segment */ // create a size sorted list if (segments.size() > 10) { if (log.isDebugEnabled()) log.debug("Merge Phase 0 : Stating"); // long[] segmentSize = new long[segments.size() - 1]; // File[] segmentName = new File[segments.size() - 1]; for (Iterator<SegmentInfo> i = segments.iterator(); i.hasNext();) { i.next().loadSize(); } Collections.sort(segments, new Comparator<SegmentInfo>() { public int compare(SegmentInfo o1, SegmentInfo o2) { long l = o1.getSize() - o2.getSize(); if (l == 0) { return 0; } else if (l < 0) { return -1; } else { return 1; } } }); long sizeBlock = 0; int ninblock = 0; int mergegroupno = 1; int[] mergegroup = new int[segments.size()]; int[] groupstomerge = new int[segments.size()]; mergegroup[0] = mergegroupno; { int j = 0; for (int i = 0; i < mergegroup.length; i++) { if (segments.get(i).getSize() < maxMegeSegmentSize) { groupstomerge[i] = 0; if (ninblock == 0) { sizeBlock = segments.get(0).getSize(); ninblock = 1; if (log.isDebugEnabled()) log.debug("Order Size = " + sizeBlock); } if (segments.get(i).getSize() > sizeBlock / 10) { ninblock++; // count up blocks that have the same order of // size } else { // if there are more than 2 in the block force a // merge if (ninblock >= 2) { groupstomerge[j++] = mergegroupno; } // reset for the next order of magnitude down ninblock = 1; mergegroupno++; sizeBlock = segments.get(i).getSize(); } mergegroup[i] = mergegroupno; } } // catch the merge all case if (ninblock >= 2) { groupstomerge[j++] = mergegroupno; } if (j > 0) { StringBuilder status = new StringBuilder(); for (int i = 0; i < segments.size(); i++) { SegmentInfo si = segments.get(i); status.append("Segment ").append(i).append(" n").append(si.getName()).append(" s") .append(si.getSize()).append(" g").append(mergegroup[i]).append("\n"); } for (int i = 0; i < groupstomerge.length; i++) { status.append("Merge group ").append(i).append(" m").append(groupstomerge[i]) .append("\n"); } if (log.isDebugEnabled()) log.debug("Search Merge \n" + status); } } // groups to merge contains a list of group numbers that need to // be // merged. // mergegroup marks each segment with a group number. for (int i = 0; i < groupstomerge.length; i++) { if (groupstomerge[i] != 0) { StringBuilder status = new StringBuilder(); status.append("Group ").append(i).append(" Merge ").append(groupstomerge[i]).append("\n"); // merge the old segments into a new segment. SegmentInfo mergeSegment = clusterFS.newSegment(); IndexWriter mergeIndexWriter = null; boolean mergeOk = false; try { mergeIndexWriter = new IndexWriter(FSDirectory.open(mergeSegment.getSegmentLocation()), getAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); mergeIndexWriter.setUseCompoundFile(true); // indexWriter.setInfoStream(System.out); mergeIndexWriter.setMaxMergeDocs(50); mergeIndexWriter.setMergeFactor(50); List<Directory> indexes = new ArrayList<Directory>(); long currentSize = 0L; for (int j = 0; j < mergegroup.length; j++) { // find if this segment is in the current merge // group SegmentInfo si = segments.get(j); if (mergegroup[j] == groupstomerge[i]) { // if we merge this segment will the result // probably remain small enough if (si.isDeleted()) { status.append(" Skipped, Segment is already deleted ").append(" ") .append(si.getName()).append(" || ").append(mergeSegment.getName()) .append("\n"); } else if ((currentSize + si.getSize()) < maxSegmentSize) { currentSize += si.getSize(); Directory d = FSDirectory.open(si.getSegmentLocation()); if (d.fileExists("segments.gen")) { status.append(" Merge ").append(si.getName()).append(" >> ") .append(mergeSegment.getName()).append("\n"); indexes.add(d); } else { status.append(" Ignored segment as it does not exist ") .append(mergeSegment.getName()).append("\n"); } } else { status.append(" Skipped, size > ").append(maxSegmentSize).append(" ") .append(si.getName()).append(" || ").append(mergeSegment.getName()) .append("\n"); // Dont merge this segment this time mergegroup[j] = -10; } } } // merge in the list of segments that we have // waiting to be merged if (diagnostics) { log.info("Merging \n" + status); } mergeIndexWriter.addIndexesNoOptimize( (Directory[]) indexes.toArray(new Directory[indexes.size()])); mergeIndexWriter.optimize(); if (diagnostics) { log.info("Merged Segment contians " + mergeIndexWriter.maxDoc() + " documents "); } // mark the segment as create and ready of upload mergeSegment.setCreated(); mergeSegment.touchSegment(); if (log.isDebugEnabled()) log.debug("Done " + groupstomerge[i]); // remove old segments mergeOk = true; } catch (IOException ex) { log.error("Failed to merge search segments " + ex.getMessage()); try { clusterFS.removeLocalSegment(mergeSegment); } catch (Exception ex2) { log.error("Failed to remove merge segment " + mergeSegment.getName() + " " + ex2.getMessage()); } } finally { try { mergeIndexWriter.close(); } catch (Exception ex) { } } if (mergeOk) { for (int j = 0; j < mergegroup.length; j++) { if (mergegroup[j] == groupstomerge[i]) { clusterFS.removeLocalSegment(segments.get(j)); } } } } } } } else { log.debug("Merge Not requested "); } clusterFS.removeTemporarySegment(); clusterFS.saveSegments(); if (log.isDebugEnabled()) log.debug("+++++++++++++++++++++++++++++++++++++End Index Cycle"); } public void setRecoverCorruptedIndex(boolean recover) { } /** * @return Returns the clusterFS. */ public ClusterFilesystem getClusterFS() { return clusterFS; } /** * @param clusterFS * The clusterFS to set. */ public void setClusterFS(ClusterFilesystem clusterFS) { this.clusterFS = clusterFS; } public long getLastUpdate() { return clusterFS.getLastUpdate(); } public List getSegmentInfoList() { return clusterFS.getSegmentInfoList(); } public void closeIndexReader(IndexReader indexReader) throws IOException { if (indexReader != null) { indexReader.close(); } // only update required, no merge clusterFS.getLock(); mergeAndUpdate(false); clusterFS.releaseLock(); // if a lock was aquired, the lock should be released and the indx // synchronised } public void closeIndexWriter(IndexWriter indexWrite) throws IOException { if (indexWrite != null) { indexWrite.close(); } clusterFS.getLock(); mergeAndUpdate(true); clusterFS.releaseLock(); // we should aquire a lock, merge in the index and sync } public boolean isMultipleIndexers() { return clusterFS.isMultipleIndexers(); } public void closeIndexSearcher(IndexSearcher indexSearcher) { IndexReader indexReader = indexSearcher.getIndexReader(); boolean closedAlready = false; try { if (indexReader != null) { indexReader.close(); closedAlready = true; } } catch (Exception ex) { log.error("Failed to close Index Reader " + ex.getMessage()); } try { indexSearcher.close(); } catch (Exception ex) { if (closedAlready) { log.debug("Failed to close Index Searcher " + ex.getMessage()); } else { log.error("Failed to close Index Searcher " + ex.getMessage()); } } } /** * @return the maxMegeSegmentSize */ public long getMaxMegeSegmentSize() { return maxMegeSegmentSize; } /** * @param maxMegeSegmentSize * the maxMegeSegmentSize to set */ public void setMaxMegeSegmentSize(long maxMegeSegmentSize) { log.info("Max Segment Merge Size set to " + maxMegeSegmentSize); this.maxMegeSegmentSize = maxMegeSegmentSize; } /** * @return the maxSegmentSize */ public long getMaxSegmentSize() { return maxSegmentSize; } /** * @param maxSegmentSize * the maxSegmentSize to set */ public void setMaxSegmentSize(long maxSegmentSize) { log.info("Max Segment Size set to " + maxSegmentSize); this.maxSegmentSize = maxSegmentSize; } /** * @return the segmentThreshold */ public long getSegmentThreshold() { return segmentThreshold; } /** * @param segmentThreshold * the segmentThreshold to set */ public void setSegmentThreshold(long segmentThreshold) { log.info("New Segment Size threshold set to " + segmentThreshold); this.segmentThreshold = segmentThreshold; } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.IndexStorage#centralIndexExists() */ public boolean centralIndexExists() { return clusterFS.centralIndexExists(); } public Directory getSpellDirectory() { // TODO Auto-generated method stub return null; } }