List of usage examples for org.apache.lucene.index SegmentCommitInfo sizeInBytes
long sizeInBytes
To view the source code for org.apache.lucene.index SegmentCommitInfo sizeInBytes.
Click Source Link
From source file:org.apache.mahout.text.LuceneSegmentInputFormat.java
License:Apache License
@Override public List<LuceneSegmentInputSplit> getSplits(JobContext context) throws IOException, InterruptedException { Configuration configuration = context.getConfiguration(); LuceneStorageConfiguration lucene2SeqConfiguration = new LuceneStorageConfiguration(configuration); List<LuceneSegmentInputSplit> inputSplits = Lists.newArrayList(); List<Path> indexPaths = lucene2SeqConfiguration.getIndexPaths(); for (Path indexPath : indexPaths) { ReadOnlyFileSystemDirectory directory = new ReadOnlyFileSystemDirectory(FileSystem.get(configuration), indexPath, false, configuration); SegmentInfos segmentInfos = new SegmentInfos(); segmentInfos.read(directory);/*from ww w . ja va2 s .co m*/ for (SegmentCommitInfo segmentInfo : segmentInfos) { LuceneSegmentInputSplit inputSplit = new LuceneSegmentInputSplit(indexPath, segmentInfo.info.name, segmentInfo.sizeInBytes()); inputSplits.add(inputSplit); LOG.info("Created {} byte input split for index '{}' segment {}", segmentInfo.sizeInBytes(), indexPath.toUri(), segmentInfo.info.name); } } return inputSplits; }
From source file:org.apache.mahout.text.LuceneSegmentRecordReaderTest.java
License:Apache License
@Test public void testKey() throws Exception { for (SegmentCommitInfo segmentInfo : segmentInfos) { int docId = 0; LuceneSegmentInputSplit inputSplit = new LuceneSegmentInputSplit(getIndexPath1(), segmentInfo.info.name, segmentInfo.sizeInBytes()); TaskAttemptContext context = getTaskAttemptContext(configuration, new TaskAttemptID()); recordReader.initialize(inputSplit, context); for (int i = 0; i < 500; i++) { recordReader.nextKeyValue(); //we can't be sure of the order we are getting the segments, so we have to fudge here a bit on the id, but it is either id: i or i + 500 assertTrue("i = " + i + " docId= " + docId, String.valueOf(docId).equals(recordReader.getCurrentKey().toString()) || String.valueOf(docId + 500).equals(recordReader.getCurrentKey().toString())); assertEquals(NullWritable.get(), recordReader.getCurrentValue()); docId++;/*from w w w. j a va2s . c o m*/ } } }
From source file:org.apache.mahout.text.LuceneSegmentRecordReaderTest.java
License:Apache License
@Test(expected = IllegalArgumentException.class) public void testNonExistingIdField() throws Exception { configuration = new LuceneStorageConfiguration(getConfiguration(), asList(getIndexPath1()), new Path("output"), "nonExistingId", asList(FIELD)).serialize(); SegmentCommitInfo segmentInfo = segmentInfos.iterator().next(); LuceneSegmentInputSplit inputSplit = new LuceneSegmentInputSplit(getIndexPath1(), segmentInfo.info.name, segmentInfo.sizeInBytes()); TaskAttemptContext context = getTaskAttemptContext(configuration, new TaskAttemptID()); recordReader.initialize(inputSplit, context); }
From source file:org.apache.mahout.text.LuceneSegmentRecordReaderTest.java
License:Apache License
@Test(expected = IllegalArgumentException.class) public void testNonExistingField() throws Exception { configuration = new LuceneStorageConfiguration(getConfiguration(), asList(getIndexPath1()), new Path("output"), ID_FIELD, asList("nonExistingField")).serialize(); SegmentCommitInfo segmentInfo = segmentInfos.iterator().next(); LuceneSegmentInputSplit inputSplit = new LuceneSegmentInputSplit(getIndexPath1(), segmentInfo.info.name, segmentInfo.sizeInBytes()); TaskAttemptContext context = getTaskAttemptContext(configuration, new TaskAttemptID()); recordReader.initialize(inputSplit, context); }
From source file:org.apache.sling.oakui.OakUIWebConsole.java
License:Apache License
private void analyseIndex(@Nonnull HttpServletRequest request, @Nonnull HttpServletResponse response, @Nonnull String index) throws JSONException, IOException { // load the commit info and get all the files. // Need a directory implementation that works off the location, OakDirectory isn't exported, so will have to re-create. NodeStore ns = getNodeStore();/*from w ww .j av a 2 s .c o m*/ NodeState oakIndex = ns.getRoot().getChildNode("oak:index"); NodeStoreDirectory nsDirectory = new NodeStoreDirectory(oakIndex, index); JSONObject out = new JSONObject(); int sequence = 0; for (String file : getSegments(nsDirectory.listAll())) { if (file.startsWith("segments")) { JSONObject segmentDetail = new JSONObject(); //FSDirectory fsDirectory = new SimpleFSDirectory(new File("/Users/ieb/Adobe/CQ/6.2/crx-quickstart/repository/index/lucene-1476792800724/data")); SegmentInfos segmentCommitInfos = new SegmentInfos(); try { if ("segments.gen".equals(file)) { // segments.gen is actually a pre v3 segments file, to get the current commit open without specifying the segments file. segmentCommitInfos.read(nsDirectory); } else { segmentCommitInfos.read(nsDirectory, file); } Iterator<SegmentCommitInfo> sci = segmentCommitInfos.iterator(); JSONArray commits = new JSONArray(); while (sci.hasNext()) { SegmentCommitInfo sc = sci.next(); JSONObject commitInfo = new JSONObject(); JSONArray files = new JSONArray(); for (String f : sc.files()) { files.put(f); } commitInfo.put("files", files); commitInfo.put("delcount", sc.getDelCount()); commitInfo.put("delgen", sc.getDelGen()); commitInfo.put("hasDeletions", sc.hasDeletions()); commitInfo.put("hasFieldUpdates", sc.hasFieldUpdates()); commitInfo.put("sizeInBytes", sc.sizeInBytes()); commitInfo.put("fieldInfosGen", sc.getFieldInfosGen()); commitInfo.put("nextDelGen", sc.getNextDelGen()); commitInfo.put("version", sc.info.getVersion()); commitInfo.put("diagnostics", sc.info.getDiagnostics()); commitInfo.put("doccount", sc.info.getDocCount()); commitInfo.put("name", sc.info.name); commitInfo.put("useCompoundFile", sc.info.getUseCompoundFile()); commits.put(commitInfo); } segmentDetail.put("commits", commits); segmentDetail.put("segment_sequence", sequence); segmentDetail.put("segment_name", file); } catch (CorruptIndexException e) { LOGGER.info(e.getMessage(), e); segmentDetail.put("corruption", e.getMessage()); } out.put(file, segmentDetail); sequence++; } } response.setContentType("application/json; charset=utf-8"); response.getWriter().println(out.toString(4)); }
From source file:org.apache.solr.handler.admin.SegmentsInfoRequestHandler.java
License:Apache License
private SimpleOrderedMap<Object> getSegmentInfo(SegmentCommitInfo segmentCommitInfo) throws IOException { SimpleOrderedMap<Object> segmentInfoMap = new SimpleOrderedMap<>(); segmentInfoMap.add("name", segmentCommitInfo.info.name); segmentInfoMap.add("delCount", segmentCommitInfo.getDelCount()); segmentInfoMap.add("sizeInBytes", segmentCommitInfo.sizeInBytes()); segmentInfoMap.add("size", segmentCommitInfo.info.maxDoc()); Long timestamp = Long.parseLong(segmentCommitInfo.info.getDiagnostics().get("timestamp")); segmentInfoMap.add("age", new Date(timestamp)); segmentInfoMap.add("source", segmentCommitInfo.info.getDiagnostics().get("source")); return segmentInfoMap; }
From source file:org.elasticsearch.index.engine.internal.AsynchronousEngine.java
License:Apache License
@Override public List<Segment> segments() { try (InternalLock _ = readLock.acquire()) { ensureOpen();/*from w w w .jav a 2 s .c om*/ Map<String, Segment> segments = new HashMap<>(); // first, go over and compute the search ones... Searcher searcher = acquireSearcher("segments"); try { for (AtomicReaderContext reader : searcher.reader().leaves()) { assert reader.reader() instanceof SegmentReader; SegmentCommitInfo info = SegmentReaderUtils.segmentReader(reader.reader()).getSegmentInfo(); assert !segments.containsKey(info.info.name); Segment segment = new Segment(info.info.name); segment.search = true; segment.docCount = reader.reader().numDocs(); segment.delDocCount = reader.reader().numDeletedDocs(); segment.version = info.info.getVersion(); segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { logger.trace("failed to get size for [{}]", e, info.info.name); } segment.memoryInBytes = getReaderRamBytesUsed(reader); segments.put(info.info.name, segment); } } finally { searcher.close(); } // now, correlate or add the committed ones... if (lastCommittedSegmentInfos != null) { SegmentInfos infos = lastCommittedSegmentInfos; for (SegmentCommitInfo info : infos) { Segment segment = segments.get(info.info.name); if (segment == null) { segment = new Segment(info.info.name); segment.search = false; segment.committed = true; segment.docCount = info.info.getDocCount(); segment.delDocCount = info.getDelCount(); segment.version = info.info.getVersion(); segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { logger.trace("failed to get size for [{}]", e, info.info.name); } segments.put(info.info.name, segment); } else { segment.committed = true; } } } Segment[] segmentsArr = segments.values().toArray(new Segment[segments.values().size()]); Arrays.sort(segmentsArr, new Comparator<Segment>() { @Override public int compare(Segment o1, Segment o2) { return (int) (o1.getGeneration() - o2.getGeneration()); } }); // fill in the merges flag Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges(); for (OnGoingMerge onGoingMerge : onGoingMerges) { for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) { for (Segment segment : segmentsArr) { if (segment.getName().equals(segmentInfoPerCommit.info.name)) { segment.mergeId = onGoingMerge.getId(); break; } } } } return Arrays.asList(segmentsArr); } }
From source file:org.elasticsearch.index.engine.internal.InternalEngine.java
License:Apache License
@Override public List<Segment> segments() { rwl.readLock().lock();// www . j av a 2 s .com try { ensureOpen(); Map<String, Segment> segments = new HashMap<String, Segment>(); // first, go over and compute the search ones... Searcher searcher = acquireSearcher("segments"); try { for (AtomicReaderContext reader : searcher.reader().leaves()) { assert reader.reader() instanceof SegmentReader; SegmentCommitInfo info = SegmentReaderUtils.segmentReader(reader.reader()).getSegmentInfo(); assert !segments.containsKey(info.info.name); Segment segment = new Segment(info.info.name); segment.search = true; segment.docCount = reader.reader().numDocs(); segment.delDocCount = reader.reader().numDeletedDocs(); segment.version = info.info.getVersion(); segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { logger.trace("failed to get size for [{}]", e, info.info.name); } segment.memoryInBytes = getReaderRamBytesUsed(reader); segments.put(info.info.name, segment); } } finally { searcher.release(); } // now, correlate or add the committed ones... if (lastCommittedSegmentInfos != null) { SegmentInfos infos = lastCommittedSegmentInfos; for (SegmentCommitInfo info : infos) { Segment segment = segments.get(info.info.name); if (segment == null) { segment = new Segment(info.info.name); segment.search = false; segment.committed = true; segment.docCount = info.info.getDocCount(); segment.delDocCount = info.getDelCount(); segment.version = info.info.getVersion(); segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { logger.trace("failed to get size for [{}]", e, info.info.name); } segments.put(info.info.name, segment); } else { segment.committed = true; } } } Segment[] segmentsArr = segments.values().toArray(new Segment[segments.values().size()]); Arrays.sort(segmentsArr, new Comparator<Segment>() { @Override public int compare(Segment o1, Segment o2) { return (int) (o1.getGeneration() - o2.getGeneration()); } }); // fill in the merges flag Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges(); for (OnGoingMerge onGoingMerge : onGoingMerges) { for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) { for (Segment segment : segmentsArr) { if (segment.getName().equals(segmentInfoPerCommit.info.name)) { segment.mergeId = onGoingMerge.getId(); break; } } } } return Arrays.asList(segmentsArr); } finally { rwl.readLock().unlock(); } }
From source file:org.eu.bitzone.Leia.java
License:Apache License
public void showSegments(final Object commitsTable) throws Exception { final Object segTable = find("segmentsTable"); removeAll(segTable);//w ww .ja va 2 s . c o m final Object[] rows = getSelectedItems(commitsTable); if (rows == null || rows.length == 0) { showStatus("No commit point selected."); return; } final Object row = rows[0]; final IndexCommit commit = (IndexCommit) getProperty(row, "commit"); if (commit == null) { showStatus("Can't retrieve commit point (application error)"); return; } final Object segGen = find("segGen"); setString(segGen, "text", commit.getSegmentsFileName() + " (gen " + commit.getGeneration() + ")"); final String segName = commit.getSegmentsFileName(); final SegmentInfos infos = new SegmentInfos(); try { infos.read(dir, segName); } catch (final Exception e) { e.printStackTrace(); errorMsg("Error reading segment infos for '" + segName + ": " + e.toString()); return; } for (final SegmentCommitInfo si : infos.asList()) { final Object r = create("row"); add(segTable, r); Object cell = create("cell"); add(r, cell); setString(cell, "text", si.info.name); cell = create("cell"); add(r, cell); setString(cell, "text", String.valueOf(si.getDelGen())); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", String.valueOf(si.getDelCount())); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", String.valueOf(si.info.getDocCount())); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", si.info.getVersion()); cell = create("cell"); add(r, cell); setString(cell, "text", si.info.getCodec().getName()); final long size = si.sizeInBytes(); cell = create("cell"); add(r, cell); setString(cell, "text", Util.normalizeSize(size) + Util.normalizeUnit(size)); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", si.info.getUseCompoundFile() ? "Y" : "N"); putProperty(r, "si", si); } final Object diagsTable = find("diagsTable"); removeAll(diagsTable); }
From source file:org.getopt.luke.Luke.java
License:Apache License
public void showSegments(Object commitsTable) throws Exception { Object segTable = find("segmentsTable"); removeAll(segTable);//ww w .j a va2s . co m Object[] rows = getSelectedItems(commitsTable); if (rows == null || rows.length == 0) { showStatus("No commit point selected."); return; } Object row = rows[0]; IndexCommit commit = (IndexCommit) getProperty(row, "commit"); if (commit == null) { showStatus("Can't retrieve commit point (application error)"); return; } Object segGen = find("segGen"); setString(segGen, "text", commit.getSegmentsFileName() + " (gen " + commit.getGeneration() + ")"); String segName = commit.getSegmentsFileName(); SegmentInfos infos = new SegmentInfos(); try { infos.read(dir, segName); } catch (Exception e) { e.printStackTrace(); errorMsg("Error reading segment infos for '" + segName + ": " + e.toString()); return; } for (SegmentCommitInfo si : infos.asList()) { Object r = create("row"); add(segTable, r); Object cell = create("cell"); add(r, cell); setString(cell, "text", si.info.name); cell = create("cell"); add(r, cell); setString(cell, "text", String.valueOf(si.getDelGen())); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", String.valueOf(si.getDelCount())); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", String.valueOf(si.info.getDocCount())); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", si.info.getVersion().toString()); cell = create("cell"); add(r, cell); setString(cell, "text", si.info.getCodec().getName()); long size = si.sizeInBytes(); cell = create("cell"); add(r, cell); setString(cell, "text", Util.normalizeSize(size) + Util.normalizeUnit(size)); setChoice(cell, "alignment", "right"); cell = create("cell"); add(r, cell); setString(cell, "text", si.info.getUseCompoundFile() ? "Y" : "N"); putProperty(r, "si", si); } Object diagsTable = find("diagsTable"); removeAll(diagsTable); }