List of usage examples for org.apache.lucene.store IOContext READ
IOContext READ
To view the source code for org.apache.lucene.store IOContext READ.
Click Source Link
From source file:edu.illinois.cs.cogcomp.bigdata.lucene.Lucene.java
License:Open Source License
public static IndexReader ramReader(String pathToIndex) throws IOException { return DirectoryReader.open(new RAMDirectory(new MMapDirectory(Paths.get(pathToIndex)), IOContext.READ)); }
From source file:org.apache.blur.filter.IndexFileBitSet.java
License:Apache License
public void load() throws IOException { String fileName = getFileName(); _indexInput = _directory.openInput(fileName, IOContext.READ); int words = (_numBits / 64) + 1; int correctLength = words * 8; long length = _indexInput.length(); if (correctLength != length) { throw new IOException("File [" + fileName + "] with length [" + length + "] does not match correct length of [" + correctLength + "]"); }/*from w w w . j a v a 2s. c om*/ }
From source file:org.apache.blur.filter.IndexFileBitSet.java
License:Apache License
public void create(DocIdSetIterator it) throws IOException { String fileName = getFileName(); if (_directory.fileExists(getFileName())) { LOG.warn("Filter [{0}] in directory [{1}] being recreated due to incorrect size.", fileName, _directory);//from ww w . j a va 2s . c o m _directory.deleteFile(fileName); } IndexOutput output = _directory.createOutput(fileName, IOContext.READ); int index; int currentWordNum = 0; long wordValue = 0; while ((index = it.nextDoc()) < _numBits) { int wordNum = index >> 6; // div 64 if (currentWordNum > wordNum) { throw new IOException("We got a problem here!"); } while (currentWordNum < wordNum) { output.writeLong(wordValue); currentWordNum++; wordValue = 0; } int bit = index & 0x3f; // mod 64 long bitmask = 1L << bit; wordValue |= bitmask; } if (_numBits > 0) { int totalWords = (_numBits / 64) + 1; while (currentWordNum < totalWords) { output.writeLong(wordValue); currentWordNum++; wordValue = 0; } } output.close(); }
From source file:org.apache.blur.lucene.warmup.IndexWarmup.java
License:Apache License
private void warm(IndexReader reader, String segmentName, String fileName, String fieldName, long startingPosition, long endingPosition, String context) throws IOException { Directory dir = getDirectory(reader, segmentName, context); if (dir == null) { LOG.info("Context [{0}] cannot find segment [{1}]", context, segmentName); return;//w w w . j a v a 2 s . c om } if (endingPosition == Long.MAX_VALUE) { endingPosition = dir.fileLength(fileName) - 1; } if (_isClosed.get()) { LOG.info("Context [{0}] index closed", context); return; } long length = endingPosition - startingPosition; final long totalLength = length; IndexInput input = new ThrottledIndexInput(dir.openInput(fileName, IOContext.READ), _maxBytesPerSec); try { input.seek(startingPosition); byte[] buf = new byte[8192]; long start = System.nanoTime(); long bytesReadPerPass = 0; while (length > 0) { long now = System.nanoTime(); if (start + _5_SECONDS < now) { double seconds = (now - start) / 1000000000.0; double rateMbPerSec = (bytesReadPerPass / seconds) / 1000 / 1000; double complete = (((double) totalLength - (double) length) / (double) totalLength) * 100.0; LOG.debug( "Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at rate of [{4} MB/s]", fieldName, fileName, complete, context, rateMbPerSec); start = System.nanoTime(); bytesReadPerPass = 0; if (_isClosed.get()) { LOG.info("Context [{0}] index closed", context); return; } } int len = (int) Math.min(length, buf.length); input.readBytes(buf, 0, len); length -= len; bytesReadPerPass += len; } long now = System.nanoTime(); double seconds = (now - start) / 1000000000.0; if (seconds < 1) { seconds = 1; } double rateMbPerSec = (bytesReadPerPass / seconds) / 1000 / 1000; LOG.debug("Context [{3}] warming field [{0}] in file [{1}] is [{2}%] complete at rate of [{4} MB/s]", fieldName, fileName, 100, context, rateMbPerSec); } finally { input.close(); } }
From source file:org.apache.blur.mapreduce.lib.GenericRecordReader.java
License:Apache License
public void initialize(BlurInputSplit blurInputSplit, Configuration configuration) throws IOException { if (_setup) { return;//from w w w .ja v a 2s .c o m } _setup = true; _table = blurInputSplit.getTable(); Path localCachePath = BlurInputFormat.getLocalCachePath(configuration); List<String> files = blurInputSplit.getDirectoryFiles(); LOG.info("Local cache path [{0}]", localCachePath); _directory = BlurInputFormat.getDirectory(configuration, _table.toString(), blurInputSplit.getDir(), files); SegmentInfoPerCommit commit = segmentInfosRead(_directory, blurInputSplit.getSegmentsName(), blurInputSplit.getSegmentInfoName()); SegmentInfo segmentInfo = commit.info; if (localCachePath != null) { _readingDirectory = copyFilesLocally(configuration, _directory, _table.toString(), blurInputSplit.getDir(), localCachePath, commit.files()); } else { _readingDirectory = _directory; } Blur024Codec blur024Codec = new Blur024Codec(); IOContext iocontext = IOContext.READ; String segmentName = segmentInfo.name; FieldInfos fieldInfos = blur024Codec.fieldInfosFormat().getFieldInfosReader().read(_readingDirectory, segmentName, iocontext); if (commit.getDelCount() > 0) { _liveDocs = blur024Codec.liveDocsFormat().readLiveDocs(_readingDirectory, commit, iocontext); } _fieldsReader = blur024Codec.storedFieldsFormat().fieldsReader(_readingDirectory, segmentInfo, fieldInfos, iocontext); _maxDoc = commit.info.getDocCount(); }
From source file:org.apache.blur.mapreduce.lib.GenericRecordReader.java
License:Apache License
private SegmentInfoPerCommit segmentInfosRead(Directory directory, String segmentFileName, String segmentInfoName) throws IOException { boolean success = false; ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ)); try {//from ww w . j av a2 s . c o m final int format = input.readInt(); if (format == CodecUtil.CODEC_MAGIC) { // 4.0+ CodecUtil.checkHeaderNoMagic(input, "segments", SegmentInfos.VERSION_40, SegmentInfos.VERSION_40); input.readLong();// read version input.readInt(); // read counter int numSegments = input.readInt(); if (numSegments < 0) { throw new CorruptIndexException( "invalid segment count: " + numSegments + " (resource: " + input + ")"); } for (int seg = 0; seg < numSegments; seg++) { String segName = input.readString(); Codec codec = Codec.forName(input.readString()); SegmentInfo info = codec.segmentInfoFormat().getSegmentInfoReader().read(directory, segName, IOContext.READ); info.setCodec(codec); long delGen = input.readLong(); int delCount = input.readInt(); if (delCount < 0 || delCount > info.getDocCount()) { throw new CorruptIndexException( "invalid deletion count: " + delCount + " (resource: " + input + ")"); } if (segName.equals(segmentInfoName)) { success = true; return new SegmentInfoPerCommit(info, delCount, delGen); } } } else { throw new IOException("Legacy Infos not supported for dir [" + directory + "]."); } throw new IOException("Segment [" + segmentInfoName + "] nout found in dir [" + directory + "]"); } finally { if (!success) { IOUtils.closeWhileHandlingException(input); } else { input.close(); } } }
From source file:org.apache.blur.store.hdfs.HdfsDirectoryResourceTest.java
License:Apache License
private void executeReads(HdfsDirectory dir, String name) throws IOException, InterruptedException { IndexInput input = dir.openInput(name, IOContext.READ); assertResourceCount(1);/* www .j a va 2s.co m*/ input.readLong(); input.seek(0L); for (int i = 0; i < 2; i++) { readSeq(input.clone(), READ_SIZE); assertResourceCount(1 + i + 1); } input.close(); }
From source file:org.apache.blur.store.hdfs.MultiInstancesHdfsDirectoryTest.java
License:Apache License
@Test public void testMultiInstancesHdfsDirectoryTest1() throws IOException, InterruptedException { HdfsDirectory dir1 = new HdfsDirectory(_configuration, new Path(_root, "dir")); IndexOutput output = dir1.createOutput("a", IOContext.DEFAULT); output.writeInt(1234);/*from ww w.j av a 2 s. com*/ output.close(); HdfsDirectory dir2 = new HdfsDirectory(_configuration, new Path(_root, "dir")); IndexInput input = dir2.openInput("a", IOContext.READ); assertEquals(4, input.length()); assertEquals(1234, input.readInt()); input.close(); dir1.close(); dir2.close(); }
From source file:org.apache.blur.store.hdfs_v2.FastHdfsKeyValueDirectoryTest.java
License:Apache License
private byte[] readSegmentsGen(FastHdfsKeyValueDirectory directory) throws IOException { boolean fileExists = directory.fileExists("segments.gen"); if (!fileExists) { return null; }/*from w w w . j a v a2 s . c o m*/ IndexInput input = directory.openInput("segments.gen", IOContext.READ); byte[] data = new byte[(int) input.length()]; input.readBytes(data, 0, data.length); return data; }
From source file:org.apache.mahout.text.LuceneSegmentInputSplitTest.java
License:Apache License
private void assertSegmentContainsOneDoc(String segmentName) throws IOException { LuceneSegmentInputSplit inputSplit = new LuceneSegmentInputSplit(indexPath1, segmentName, 1000); SegmentCommitInfo segment = inputSplit.getSegment(configuration); SegmentReader segmentReader = new SegmentReader(segment, 1, IOContext.READ);//SegmentReader.get(true, segment, 1); assertEquals(segmentName, segment.info.name); assertEquals(1, segmentReader.numDocs()); }