List of usage examples for org.apache.lucene.index IndexCommit getSegmentCount
public abstract int getSegmentCount();
From source file:org.apache.solr.core.SolrDeletionPolicy.java
License:Apache License
private void updateCommits(List<? extends IndexCommit> commits) { // to be safe, we should only call delete on a commit point passed to us // in this specific call (may be across diff IndexWriter instances). // this will happen rarely, so just synchronize everything // for safety and to avoid race conditions synchronized (this) { long maxCommitAgeTimeStamp = -1L; IndexCommit newest = commits.get(commits.size() - 1); // SOLR-4547: Removed the filenames from this log entry because this // method is only called from methods that have just logged them // at DEBUG. log.info("newest commit generation = " + newest.getGeneration()); int singleSegKept = (newest.getSegmentCount() == 1) ? 1 : 0; int totalKept = 1; // work our way from newest to oldest, skipping the first since we always want to keep it. for (int i = commits.size() - 2; i >= 0; i--) { IndexCommit commit = commits.get(i); // delete anything too old, regardless of other policies try { if (maxCommitAge != null) { if (maxCommitAgeTimeStamp == -1) { DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.ROOT); maxCommitAgeTimeStamp = dmp.parseMath(maxCommitAge).getTime(); }/*from www.j a v a 2 s.co m*/ if (IndexDeletionPolicyWrapper.getCommitTimestamp(commit) < maxCommitAgeTimeStamp) { commit.delete(); continue; } } } catch (Exception e) { log.warn("Exception while checking commit point's age for deletion", e); } if (singleSegKept < maxOptimizedCommitsToKeep && commit.getSegmentCount() == 1) { totalKept++; singleSegKept++; continue; } if (totalKept < maxCommitsToKeep) { totalKept++; continue; } commit.delete(); } } // end synchronized }
From source file:org.apache.solr.handler.ReplicationHandler.java
License:Apache License
@Override @SuppressWarnings("unchecked") public void inform(SolrCore core) { this.core = core; registerFileStreamResponseWriter();//from w w w .j a v a2s . c o m registerCloseHook(); Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM); if (nbtk != null) { numberBackupsToKeep = Integer.parseInt(nbtk.toString()); } else { numberBackupsToKeep = 0; } NamedList slave = (NamedList) initArgs.get("slave"); boolean enableSlave = isEnabled(slave); if (enableSlave) { tempSnapPuller = snapPuller = new SnapPuller(slave, this, core); isSlave = true; } NamedList master = (NamedList) initArgs.get("master"); boolean enableMaster = isEnabled(master); if (enableMaster || enableSlave) { if (core.getCoreDescriptor().getCoreContainer().getZkController() != null) { LOG.warn("SolrCloud is enabled for core " + core.getName() + " but so is old-style replication. Make sure you" + " intend this behavior, it usually indicates a mis-configuration. Master setting is " + Boolean.toString(enableMaster) + " and slave setting is " + Boolean.toString(enableSlave)); } } if (!enableSlave && !enableMaster) { enableMaster = true; master = new NamedList<Object>(); } if (enableMaster) { includeConfFiles = (String) master.get(CONF_FILES); if (includeConfFiles != null && includeConfFiles.trim().length() > 0) { List<String> files = Arrays.asList(includeConfFiles.split(",")); for (String file : files) { if (file.trim().length() == 0) continue; String[] strs = file.trim().split(":"); // if there is an alias add it or it is null confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); } LOG.info("Replication enabled for following config files: " + includeConfFiles); } List backup = master.getAll("backupAfter"); boolean backupOnCommit = backup.contains("commit"); boolean backupOnOptimize = !backupOnCommit && backup.contains("optimize"); List replicateAfter = master.getAll(REPLICATE_AFTER); replicateOnCommit = replicateAfter.contains("commit"); replicateOnOptimize = !replicateOnCommit && replicateAfter.contains("optimize"); if (!replicateOnCommit && !replicateOnOptimize) { replicateOnCommit = true; } // if we only want to replicate on optimize, we need the deletion policy to // save the last optimized commit point. if (replicateOnOptimize) { IndexDeletionPolicyWrapper wrapper = core.getDeletionPolicy(); IndexDeletionPolicy policy = wrapper == null ? null : wrapper.getWrappedDeletionPolicy(); if (policy instanceof SolrDeletionPolicy) { SolrDeletionPolicy solrPolicy = (SolrDeletionPolicy) policy; if (solrPolicy.getMaxOptimizedCommitsToKeep() < 1) { solrPolicy.setMaxOptimizedCommitsToKeep(1); } } else { LOG.warn("Replication can't call setMaxOptimizedCommitsToKeep on " + policy); } } if (replicateOnOptimize || backupOnOptimize) { core.getUpdateHandler() .registerOptimizeCallback(getEventListener(backupOnOptimize, replicateOnOptimize)); } if (replicateOnCommit || backupOnCommit) { replicateOnCommit = true; core.getUpdateHandler().registerCommitCallback(getEventListener(backupOnCommit, replicateOnCommit)); } if (replicateAfter.contains("startup")) { replicateOnStart = true; RefCounted<SolrIndexSearcher> s = core.getNewestSearcher(false); try { DirectoryReader reader = s == null ? null : s.get().getIndexReader(); if (reader != null && reader.getIndexCommit() != null && reader.getIndexCommit().getGeneration() != 1L) { try { if (replicateOnOptimize) { Collection<IndexCommit> commits = DirectoryReader.listCommits(reader.directory()); for (IndexCommit ic : commits) { if (ic.getSegmentCount() == 1) { if (indexCommitPoint == null || indexCommitPoint.getGeneration() < ic.getGeneration()) indexCommitPoint = ic; } } } else { indexCommitPoint = reader.getIndexCommit(); } } finally { // We don't need to save commit points for replication, the SolrDeletionPolicy // always saves the last commit point (and the last optimized commit point, if needed) /*** if(indexCommitPoint != null){ core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration()); } ***/ } } // ensure the writer is init'd so that we have a list of commit points RefCounted<IndexWriter> iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); iw.decref(); } catch (IOException e) { LOG.warn("Unable to get IndexCommit on startup", e); } finally { if (s != null) s.decref(); } } String reserve = (String) master.get(RESERVE); if (reserve != null && !reserve.trim().equals("")) { reserveCommitDuration = SnapPuller.readInterval(reserve); } LOG.info("Commits will be reserved for " + reserveCommitDuration); isMaster = true; } }
From source file:org.eu.bitzone.Leia.java
License:Apache License
/** * Initialize index overview and other GUI elements. This method is called always when a new index is open. *//*from w w w. j a v a2 s. c o m*/ private void initOverview() { try { initSolrTypes(); courier = new Font("Courier", getFont().getStyle(), getFont().getSize()); lastST = find("lastST"); setBoolean(find("bReload"), "enabled", true); setBoolean(find("bClose"), "enabled", true); setBoolean(find("bCommit"), "enabled", true); final Object sTable = find("sTable"); removeAll(sTable); final Object docTable = find("docTable"); removeAll(docTable); final Object cbType = find("cbType"); populateAnalyzers(cbType); final Object pOver = find("pOver"); Object iName = find("idx"); String idxName; if (pName.length() > 40) { idxName = pName.substring(0, 10) + "..." + pName.substring(pName.length() - 27); } else { idxName = pName; } setString(iName, "text", idxName + (readOnly ? " (R)" : "")); iName = find(pOver, "iName"); setString(iName, "text", pName + (readOnly ? " (Read-Only)" : "")); final Object dirImpl = find("dirImpl"); String implName = "N/A"; if (dir == null) { if (ir != null) { implName = "N/A (reader is " + ir.getClass().getName() + ")"; } } else { implName = dir.getClass().getName(); } setString(dirImpl, "text", implName); final Object fileSize = find("iFileSize"); final long totalFileSize = Util.calcTotalFileSize(pName, dir); setString(fileSize, "text", Util.normalizeSize(totalFileSize) + Util.normalizeUnit(totalFileSize)); final Object iFormat = find(pOver, "iFormat"); final Object iCaps = find(pOver, "iCaps"); String formatText = "N/A"; String formatCaps = "N/A"; if (dir != null) { final IndexGate.FormatDetails formatDetails = IndexGate.getIndexFormat(dir); formatText = formatDetails.genericName; formatCaps = formatDetails.capabilities; } setString(iFormat, "text", formatText); setString(iCaps, "text", formatCaps); if (ir == null) { return; } // we need IndexReader from now on idxInfo = new IndexInfo(ir, pName); Object iDocs = find(pOver, "iDocs"); final String numdocs = String.valueOf(ir.numDocs()); setString(iDocs, "text", numdocs); iDocs = find("iDocs1"); setString(iDocs, "text", String.valueOf(ir.maxDoc() - 1)); final Object iFields = find(pOver, "iFields"); fn = idxInfo.getFieldNames(); if (fn.size() == 0) { showStatus("Empty index."); } showFiles(dir, null); if (ir instanceof CompositeReader) { ar = SlowCompositeReaderWrapper.wrap((CompositeReader) ir); } else if (ir instanceof AtomicReader) { ar = (AtomicReader) ir; } if (ar != null) { infos = ar.getFieldInfos(); } showCommits(); final Object fList = find(pOver, "fList"); final Object defFld = find("defFld"); final Object fCombo = find("fCombo"); final TreeSet<String> fields = new TreeSet<String>(fn); idxFields = fields.toArray(new String[fields.size()]); setString(iFields, "text", String.valueOf(idxFields.length)); final Object iTerms = find(pOver, "iTerms"); if (!slowAccess) { final Thread t = new Thread() { @Override public void run() { final Object r = create("row"); final Object cell = create("cell"); add(r, cell); add(fList, r); setBoolean(cell, "enabled", false); setString(cell, "text", "..wait.."); try { numTerms = idxInfo.getNumTerms(); termCounts = idxInfo.getFieldTermCounts(); setString(iTerms, "text", String.valueOf(numTerms)); initFieldList(fList, fCombo, defFld); } catch (final Exception e) { e.printStackTrace(); numTerms = -1; termCounts = Collections.emptyMap(); showStatus("ERROR: can't count terms per field"); } } }; t.start(); } else { setString(iTerms, "text", "N/A"); initFieldList(fList, fCombo, defFld); } final Object iDel = find(pOver, "iDelOpt"); final String sDel = ir.hasDeletions() ? "Yes (" + ir.numDeletedDocs() + ")" : "No"; final IndexCommit ic = ir instanceof DirectoryReader ? ((DirectoryReader) ir).getIndexCommit() : null; final String opt = ic != null ? ic.getSegmentCount() == 1 ? "Yes" : "No" : "?"; final String sDelOpt = sDel + " / " + opt; setString(iDel, "text", sDelOpt); final Object iVer = find(pOver, "iVer"); String verText = "N/A"; if (ic != null) { verText = Long.toHexString(((DirectoryReader) ir).getVersion()); } setString(iVer, "text", verText); final Object iTiiDiv = find(pOver, "iTiiDiv"); String divText = "N/A"; if (ir instanceof DirectoryReader) { final List<AtomicReaderContext> contexts = ir.leaves(); if (contexts.size() > 0 && contexts.get(0).reader() instanceof SegmentReader) { divText = String.valueOf(((SegmentReader) contexts.get(0).reader()).getTermInfosIndexDivisor()); } } setString(iTiiDiv, "text", divText); final Object iCommit = find(pOver, "iCommit"); String commitText = "N/A"; if (ic != null) { commitText = ic.getSegmentsFileName() + " (generation=" + ic.getGeneration() + ", segs=" + ic.getSegmentCount() + ")"; } setString(iCommit, "text", commitText); final Object iUser = find(pOver, "iUser"); String userData = null; if (ic != null) { final Map<String, String> userDataMap = ic.getUserData(); if (userDataMap != null && !userDataMap.isEmpty()) { userData = userDataMap.toString(); } else { userData = "--"; } } else { userData = "(not supported)"; } setString(iUser, "text", userData); final Object nTerms = find("nTerms"); if (!slowAccess) { final Thread t = new Thread() { @Override public void run() { actionTopTerms(nTerms); } }; t.start(); } } catch (final Exception e) { e.printStackTrace(); errorMsg(e.getMessage()); } }
From source file:org.eu.bitzone.Leia.java
License:Apache License
private void showCommits() throws Exception { final Object commitsTable = find("commitsTable"); removeAll(commitsTable);//from w w w.j av a2s . c o m if (dir == null) { final Object row = create("row"); final Object cell = create("cell"); setString(cell, "text", "<not available>"); setBoolean(cell, "enabled", false); add(row, cell); add(commitsTable, row); return; } final List<IndexCommit> commits = DirectoryReader.listCommits(dir); final IndexCommit current = ir instanceof DirectoryReader ? ((DirectoryReader) ir).getIndexCommit() : null; // commits are ordered from oldest to newest ? final Iterator<IndexCommit> it = commits.iterator(); int rowNum = 0; while (it.hasNext()) { final IndexCommit commit = it.next(); // figure out the name of the segment files final Collection<String> files = commit.getFileNames(); final Iterator<String> itf = files.iterator(); final Object row = create("row"); final boolean enabled = rowNum < commits.size() - 1; final Color color = Color.BLUE; rowNum++; add(commitsTable, row); putProperty(row, "commit", commit); if (enabled) { putProperty(row, "commitDeletable", Boolean.TRUE); } Object cell = create("cell"); final String gen = String.valueOf(commit.getGeneration()); setString(cell, "text", gen); add(row, cell); cell = create("cell"); setString(cell, "text", commit.isDeleted() ? "Y" : "N"); add(row, cell); cell = create("cell"); setString(cell, "text", String.valueOf(commit.getSegmentCount())); add(row, cell); cell = create("cell"); final Map userData = commit.getUserData(); if (userData != null && !userData.isEmpty()) { setString(cell, "text", userData.toString()); } else { setString(cell, "text", ""); } add(row, cell); if (commit.equals(current)) { final Object[] cells = getItems(row); for (final Object c : cells) { setColor(c, "foreground", color); } } } }
From source file:org.getopt.luke.Luke.java
License:Apache License
/** * Initialize index overview and other GUI elements. This method is called always * when a new index is open.// w ww . j a va2s .c om * */ private void initOverview() { try { initSolrTypes(); courier = new Font("Courier", getFont().getStyle(), getFont().getSize()); lastST = find("lastST"); setBoolean(find("bReload"), "enabled", true); setBoolean(find("bClose"), "enabled", true); setBoolean(find("bCommit"), "enabled", true); Object sTable = find("sTable"); removeAll(sTable); Object docTable = find("docTable"); removeAll(docTable); Object cbType = find("cbType"); populateAnalyzers(cbType); Object pOver = find("pOver"); Object iName = find("idx"); String idxName; if (pName.length() > 40) { idxName = pName.substring(0, 10) + "..." + pName.substring(pName.length() - 27); } else { idxName = pName; } setString(iName, "text", idxName + (readOnly ? " (R)" : "")); iName = find(pOver, "iName"); setString(iName, "text", pName + (readOnly ? " (Read-Only)" : "")); Object dirImpl = find("dirImpl"); String implName = "N/A"; if (dir == null) { if (ir != null) { implName = "N/A (reader is " + ir.getClass().getName() + ")"; } } else { implName = dir.getClass().getName(); } setString(dirImpl, "text", implName); Object fileSize = find("iFileSize"); long totalFileSize = Util.calcTotalFileSize(pName, dir); setString(fileSize, "text", Util.normalizeSize(totalFileSize) + Util.normalizeUnit(totalFileSize)); Object iFormat = find(pOver, "iFormat"); Object iCaps = find(pOver, "iCaps"); String formatText = "N/A"; String formatCaps = "N/A"; if (dir != null) { IndexGate.FormatDetails formatDetails = IndexGate.getIndexFormat(dir); formatText = formatDetails.genericName; formatCaps = formatDetails.capabilities; } setString(iFormat, "text", formatText); setString(iCaps, "text", formatCaps); if (ir == null) { return; } // we need IndexReader from now on idxInfo = new IndexInfo(ir, pName); Object iDocs = find(pOver, "iDocs"); String numdocs = String.valueOf(ir.numDocs()); setString(iDocs, "text", numdocs); iDocs = find("iDocs1"); setString(iDocs, "text", String.valueOf(ir.maxDoc() - 1)); Object iFields = find(pOver, "iFields"); fn = idxInfo.getFieldNames(); if (fn.size() == 0) { showStatus("Empty index."); } showFiles(dir, null); if (ir instanceof CompositeReader) { ar = SlowCompositeReaderWrapper.wrap(ir); } else if (ir instanceof AtomicReader) { ar = (AtomicReader) ir; } if (ar != null) { infos = ar.getFieldInfos(); } showCommits(); final Object fList = find(pOver, "fList"); final Object defFld = find("defFld"); final Object fCombo = find("fCombo"); idxFields = fn.toArray(new String[fn.size()]); setString(iFields, "text", String.valueOf(idxFields.length)); final Object iTerms = find(pOver, "iTerms"); if (!slowAccess) { Thread t = new Thread() { public void run() { Object r = create("row"); Object cell = create("cell"); add(r, cell); add(fList, r); setBoolean(cell, "enabled", false); setString(cell, "text", "..wait.."); try { numTerms = idxInfo.getNumTerms(); termCounts = idxInfo.getFieldTermCounts(); setString(iTerms, "text", String.valueOf(numTerms)); initFieldList(fList, fCombo, defFld); } catch (Exception e) { e.printStackTrace(); numTerms = -1; termCounts = Collections.emptyMap(); showStatus("ERROR: can't count terms per field"); setString(cell, "text", e.getMessage()); } } }; t.start(); } else { setString(iTerms, "text", "N/A"); initFieldList(fList, fCombo, defFld); } Object iDel = find(pOver, "iDelOpt"); String sDel = ir.hasDeletions() ? "Yes (" + ir.numDeletedDocs() + ")" : "No"; IndexCommit ic = ir instanceof DirectoryReader ? ((DirectoryReader) ir).getIndexCommit() : null; String opt = ic != null ? (ic.getSegmentCount() == 1 ? "Yes" : "No") : "?"; String sDelOpt = sDel + " / " + opt; setString(iDel, "text", sDelOpt); Object iVer = find(pOver, "iVer"); String verText = "N/A"; if (ic != null) { verText = Long.toHexString(((DirectoryReader) ir).getVersion()); } setString(iVer, "text", verText); Object iTiiDiv = find(pOver, "iTiiDiv"); String divText = "N/A"; if (ir instanceof DirectoryReader) { List<AtomicReaderContext> readers = ((DirectoryReader) ir).leaves(); //.getSequentialSubReaders(); if (readers.size() > 0) { if (readers.get(0).reader() instanceof SegmentReader) { divText = String .valueOf(((SegmentReader) readers.get(0).reader()).getTermInfosIndexDivisor()); } } } setString(iTiiDiv, "text", divText); Object iCommit = find(pOver, "iCommit"); String commitText = "N/A"; if (ic != null) { commitText = ic.getSegmentsFileName() + " (generation=" + ic.getGeneration() + ", segs=" + ic.getSegmentCount() + ")"; } setString(iCommit, "text", commitText); Object iUser = find(pOver, "iUser"); String userData = null; if (ic != null) { Map<String, String> userDataMap = ic.getUserData(); if (userDataMap != null && !userDataMap.isEmpty()) { userData = userDataMap.toString(); } else { userData = "--"; } } else { userData = "(not supported)"; } setString(iUser, "text", userData); final Object nTerms = find("nTerms"); if (!slowAccess) { Thread t = new Thread() { public void run() { actionTopTerms(nTerms); } }; t.start(); } } catch (Exception e) { e.printStackTrace(); errorMsg(e.getMessage()); } }
From source file:org.getopt.luke.Luke.java
License:Apache License
private void showCommits() throws Exception { Object commitsTable = find("commitsTable"); removeAll(commitsTable);/* w w w.j av a2s . c om*/ if (dir == null) { Object row = create("row"); Object cell = create("cell"); setString(cell, "text", "<not available>"); setBoolean(cell, "enabled", false); add(row, cell); add(commitsTable, row); return; } List<IndexCommit> commits = DirectoryReader.listCommits(dir); IndexCommit current = ir instanceof DirectoryReader ? ((DirectoryReader) ir).getIndexCommit() : null; // commits are ordered from oldest to newest ? Iterator<IndexCommit> it = commits.iterator(); int rowNum = 0; while (it.hasNext()) { IndexCommit commit = (IndexCommit) it.next(); // figure out the name of the segment files Collection<String> files = commit.getFileNames(); Iterator<String> itf = files.iterator(); Object row = create("row"); boolean enabled = rowNum < commits.size() - 1; Color color = Color.BLUE; rowNum++; add(commitsTable, row); putProperty(row, "commit", commit); if (enabled) { putProperty(row, "commitDeletable", Boolean.TRUE); } Object cell = create("cell"); String gen = String.valueOf(commit.getGeneration()); setString(cell, "text", gen); add(row, cell); cell = create("cell"); setString(cell, "text", commit.isDeleted() ? "Y" : "N"); add(row, cell); cell = create("cell"); setString(cell, "text", String.valueOf(commit.getSegmentCount())); add(row, cell); cell = create("cell"); Map userData = commit.getUserData(); if (userData != null && !userData.isEmpty()) { setString(cell, "text", userData.toString()); } else { setString(cell, "text", ""); } add(row, cell); if (commit.equals(current)) { Object[] cells = getItems(row); for (Object c : cells) { setColor(c, "foreground", color); } } } }