List of usage examples for org.apache.lucene.store MMapDirectory MMapDirectory
public MMapDirectory(Path path, int maxChunkSize) throws IOException
From source file:com.b2international.index.lucene.Directories.java
License:Apache License
/** * Just like {@link #openFile(File)}, but allows you to also specify a custom {@link LockFactory}. *///from w w w .j a v a2s. co m public static FSDirectory openFile(final Path path, final LockFactory lockFactory) throws IOException { if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX || Constants.MAC_OS_X) && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { return new MMapDirectory(path, lockFactory); } else if (Constants.WINDOWS) { return new SimpleFSDirectory(path, lockFactory); } else { return new NIOFSDirectory(path, lockFactory); } }
From source file:com.novartis.pcs.ontology.service.search.OntologySearchServiceImpl.java
License:Apache License
@PostConstruct public void start() { try {//from www.ja v a 2 s . c o m Analyzer analyzer = new TermNameAnalyzer(true); File indexPath = new File(path); if (!indexPath.exists()) { indexPath.mkdirs(); } directory = new MMapDirectory(indexPath, new NativeFSLockFactory()); if (MMapDirectory.UNMAP_SUPPORTED) { ((MMapDirectory) directory).setUseUnmap(true); } boolean indexExists = IndexReader.indexExists(directory); writer = new IndexWriter(directory, analyzer, new IndexWriter.MaxFieldLength(MAX_CHARS)); if (!indexExists) { logger.info("Building ontology search index."); Collection<Term> terms = termDAO.loadAll(); for (Term term : terms) { if (StatusChecker.isValid(term)) { Collection<Document> docs = createDocuments(term); for (Document doc : docs) { writer.addDocument(doc); } } } } writer.optimize(); writer.commit(); numberOfDocuments = writer.numDocs(); xar = new LuceneIndexWriterXAResource(writer, this); reader = IndexReader.open(directory, true); searcher = new IndexSearcher(reader); rwlock = new ReentrantReadWriteLock(); } catch (Exception e) { logger.log(Level.WARNING, "Failed to start Lucene term searcher", e); stop(); throw new RuntimeException("Failed to start Lucene term searcher", e); } }
From source file:com.zimbra.cs.index.LuceneDirectory.java
License:Open Source License
/** * Creates a new {@link LuceneDirectory} with {@code SingleInstanceLockFactory}. * <p>//from w w w.j a v a 2 s. c om * You can switch Lucene's {@link FSDirectory} implementation by {@link LC#zimbra_index_lucene_io_impl}. * <ul> * <li>{@code null} -Lucene will try to pick the best {@link FSDirectory} implementation given the current * environment. Currently this returns {@link MMapDirectory} for most Solaris and Windows 64-bit JREs, * {@link NIOFSDirectory} for other non-Windows JREs, and {@link SimpleFSDirectory} for other JREs on Windows. * <li>{@code simple} - straightforward implementation using java.io.RandomAccessFile. However, it has poor * concurrent performance (multiple threads will bottleneck) as it synchronizes when multiple threads read from * the same file. * <li>{@code nio} - uses java.nio's FileChannel's positional io when reading to avoid synchronization when reading * from the same file. Unfortunately, due to a Windows-only Sun JRE bug this is a poor choice for Windows, but * on all other platforms this is the preferred choice. * <li>{@code mmap} - uses memory-mapped IO when reading. This is a good choice if you have plenty of virtual * memory relative to your index size, eg if you are running on a 64 bit JRE, or you are running on a 32 bit * JRE but your index sizes are small enough to fit into the virtual memory space. Java has currently the * limitation of not being able to unmap files from user code. The files are unmapped, when GC releases the * byte buffers. Due to this bug in Sun's JRE, MMapDirectory's IndexInput.close() is unable to close the * underlying OS file handle. Only when GC finally collects the underlying objects, which could be quite some * time later, will the file handle be closed. This will consume additional transient disk usage: on Windows, * attempts to delete or overwrite the files will result in an exception; on other platforms, which typically * have a "delete on last close" semantics, while such operations will succeed, the bytes are still consuming * space on disk. For many applications this limitation is not a problem (e.g. if you have plenty of disk * space, and you don't rely on overwriting files on Windows) but it's still an important limitation to be * aware of. This class supplies a (possibly dangerous) workaround mentioned in the bug report, which may fail * on non-Sun JVMs. * </ul> * * @param path directory path */ public static LuceneDirectory open(File path) throws IOException { String impl = LC.zimbra_index_lucene_io_impl.value(); FSDirectory dir; if ("nio".equals(impl)) { dir = new NIOFSDirectory(path, new SingleInstanceLockFactory()); } else if ("mmap".equals(impl)) { dir = new MMapDirectory(path, new SingleInstanceLockFactory()); } else if ("simple".equals(impl)) { dir = new SimpleFSDirectory(path, new SingleInstanceLockFactory()); } else { dir = FSDirectory.open(path, new SingleInstanceLockFactory()); } ZimbraLog.index.info("OpenLuceneIndex impl=%s,dir=%s", dir.getClass().getSimpleName(), path); return new LuceneDirectory(dir); }
From source file:org.elasticsearch.index.store.fs.DefaultFsDirectoryService.java
License:Apache License
@Override protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException { final MMapDirectory mmapDir = new MMapDirectory(location, lockFactory); return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) {//from w ww. j a v a 2 s . co m @Override public String[] listAll() throws IOException { // Avoid doing listAll twice: return mmapDir.listAll(); } }; }
From source file:org.elasticsearch.index.store.fs.MmapFsDirectoryService.java
License:Apache License
@Override protected FSDirectory newFSDirectory(File location, LockFactory lockFactory) throws IOException { return new MMapDirectory(location, buildLockFactory()); }
From source file:org.elasticsearch.index.store.fs.MmapFsStore.java
License:Apache License
@Inject public MmapFsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) throws IOException { super(shardId, indexSettings, indexStore); LockFactory lockFactory = buildLockFactory(); File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); location.mkdirs();//from ww w. j a v a 2s. c o m this.fsDirectory = new MMapDirectory(location, lockFactory); boolean suggestUseCompoundFile; Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache); if (switchDirectory != null) { suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; if (switchDirectory.v2() != null) { suggestUseCompoundFile = switchDirectory.v2(); } logger.debug("using [mmap_fs] store with path [{}], cache [true] with extensions [{}]", fsDirectory.getDirectory(), switchDirectory.v1().primaryExtensions()); directory = wrapDirectory(switchDirectory.v1()); } else { suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; directory = wrapDirectory(fsDirectory); logger.debug("using [mmap_fs] store with path [{}]", fsDirectory.getDirectory()); } this.suggestUseCompoundFile = suggestUseCompoundFile; }
From source file:org.elasticsearch.index.store.FsDirectoryService.java
License:Apache License
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { final String storeType = indexSettings.get(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.DEFAULT.getSettingsKey()); if (IndexStoreModule.Type.FS.match(storeType) || IndexStoreModule.Type.DEFAULT.match(storeType)) { final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults if (open instanceof MMapDirectory && Constants.WINDOWS == false) { return newDefaultDir(location, (MMapDirectory) open, lockFactory); }/* w w w . j a va2 s . co m*/ return open; } else if (IndexStoreModule.Type.SIMPLEFS.match(storeType)) { return new SimpleFSDirectory(location, lockFactory); } else if (IndexStoreModule.Type.NIOFS.match(storeType)) { return new NIOFSDirectory(location, lockFactory); } else if (IndexStoreModule.Type.MMAPFS.match(storeType)) { return new MMapDirectory(location, lockFactory); } throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); }
From source file:org.elasticsearch.index.store.smbmmapfs.SmbMmapFsDirectoryService.java
License:Apache License
@Override protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { logger.debug("wrapping MMapDirectory for SMB"); return new SmbDirectoryWrapper(new MMapDirectory(location, buildLockFactory(indexSettings))); }
From source file:org.modeshape.jcr.index.lucene.LuceneConfig.java
License:Apache License
private Directory directory(String directoryClass, String workspaceName, String indexName) throws IOException { boolean useLockFactory = lockFactory != null; if (StringUtil.isBlank(basePath)) { return useLockFactory ? new RAMDirectory(lockFactory) : new RAMDirectory(); }/*w w w .j a v a 2 s. c om*/ Path path = Paths.get(basePath, workspaceName, indexName); if (StringUtil.isBlank(directoryClass)) { return useLockFactory ? FSDirectory.open(path, lockFactory) : FSDirectory.open(path); } switch (directoryClass) { case "org.apache.lucene.store.RAMDirectory": { return useLockFactory ? new RAMDirectory(lockFactory) : new RAMDirectory(); } case "org.apache.lucene.store.MMapDirectory": { return useLockFactory ? new MMapDirectory(path, lockFactory) : new MMapDirectory(path); } case "org.apache.lucene.store.NIOFSDirectory": { return useLockFactory ? new NIOFSDirectory(path, lockFactory) : new NIOFSDirectory(path); } case "org.apache.lucene.store.SimpleFSDirectory": { return useLockFactory ? new SimpleFSDirectory(path, lockFactory) : new SimpleFSDirectory(path); } default: { throw new IllegalArgumentException("Unknown Lucene directory: " + directoryClass); } } }
From source file:org.polymap.core.runtime.recordstore.lucene.LuceneRecordStore.java
License:Open Source License
/** * Creates a new store for the given filesystem directory. * //from ww w . j av a 2s . com * @param indexDir The directory to hold the store files. * @param clean * @throws IOException */ public LuceneRecordStore(File indexDir, boolean clean) throws IOException { if (!indexDir.exists()) { indexDir.mkdirs(); } directory = null; // limit mmap memory to 1GB // seems that bigger mmap indexes slow down performance long dbSize = FileUtils.sizeOfDirectory(indexDir); long mmapLimit = Long.parseLong(System.getProperty("org.polymap.core.LuceneRecordStore.mmaplimit", "1000")) * 1000000; if (dbSize > mmapLimit) { directory = Constants.WINDOWS ? new SimpleFSDirectory(indexDir, null) : new NIOFSDirectory(indexDir, null); open(clean); } // use mmap on 32bit Linux of index size < 100MB // more shared memory results in system stall under rare conditions else if (Constants.LINUX && !Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED && dbSize < 100 * 1024 * 1024) { try { directory = new MMapDirectory(indexDir, null); open(clean); } catch (OutOfMemoryError e) { log.info("Unable to mmap index: falling back to default."); } } // default if (searcher == null) { directory = FSDirectory.open(indexDir); open(clean); } log.info("Database: " + indexDir.getAbsolutePath() + "\n size: " + FileUtils.sizeOfDirectory(indexDir) + "\n using: " + directory.getClass().getSimpleName() + "\n files in directry: " + Arrays.asList(directory.listAll())); }