List of usage examples for org.apache.lucene.store RAMDirectory RAMDirectory
public RAMDirectory()
From source file:com.nearinfinity.blur.search.TestingPagingCollector.java
License:Apache License
private static IndexReader getReaderFlatScore(int length) throws Exception { RAMDirectory directory = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer())); for (int i = 0; i < length; i++) { Document document = new Document(); document.add(new Field("f1", "value", Store.NO, Index.ANALYZED_NO_NORMS)); indexWriter.addDocument(document); }//from w w w .ja va2 s .c o m indexWriter.close(); return IndexReader.open(directory); }
From source file:com.netcrest.pado.index.provider.lucene.LuceneBuilderRAMDirectory.java
License:Open Source License
@SuppressWarnings({ "unchecked", "rawtypes", "resource" })
public void buildTemporalKeys(boolean createNewDirectory) {
Cache cache = CacheFactory.getAnyInstance();
Region<String, RAMDirectory> region = cache
.getRegion(IndexMatrixUtil.getProperty(Constants.PROP_REGION_LUCENE));
TemporalType[] temporalTypes = GemfireTemporalManager.getAllTemporalTypes();
for (TemporalType type : temporalTypes) {
IndexWriter writer = null;/* w w w. jav a 2 s . c o m*/
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer);
iwc.setOpenMode(OpenMode.CREATE);
LuceneField luceneBuilder = new LuceneField();
LuceneSearch ls = LuceneSearch.getLuceneSearch(type.getFullPath());
StandardQueryParser parser = ls.createParser();
TemporalManager tm = TemporalManager.getTemporalManager(type.getFullPath());
try {
List<?> identityKeyList = tm.getIdentityKeyList();
if (identityKeyList.size() == 0) {
continue;
}
RAMDirectory directory;
if (createNewDirectory) {
directory = new RAMDirectory();
} else {
directory = region.get(type.getFullPath());
if (directory == null) {
directory = new RAMDirectory();
}
}
writer = new IndexWriter(directory, iwc);
// determine the identity key type, public fields and getters
Field fields[] = null;
Method getters[] = null;
Class keyType = null;
for (Object key : identityKeyList) {
if (ReflectionHelper.isPrimitiveWrapper(key.getClass())) {
fields = null;
getters = null;
keyType = key.getClass();
} else {
fields = ReflectionHelper.getPublicFields(key.getClass());
getters = ReflectionHelper.getPublicGetters(key.getClass());
}
break;
}
if (keyType != null) {
configNumericType(parser, "IdentityKey", keyType);
// primitive
List<Document> docList = new ArrayList();
if (keyType == String.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", key.toString()));
docList.add(doc);
}
} else if (keyType == Integer.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Integer) key));
docList.add(doc);
}
} else if (keyType == Long.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Long) key));
docList.add(doc);
}
} else if (keyType == Float.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Float) key));
docList.add(doc);
}
} else if (keyType == Double.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Double) key));
docList.add(doc);
}
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning(ex);
}
} else {
try {
// fields
if (fields != null && fields.length > 0) {
// configure numeric types
for (Field field : fields) {
configNumericType(parser, field.getName(), field.getType());
}
List<Document> docList = new ArrayList();
for (Object key : identityKeyList) {
Document doc = luceneBuilder.createDocument();
for (Field field : fields) {
Object obj = field.get(key);
Class fieldType = field.getType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(field.getName(), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(field.getName(), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(field.getName(), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(field.getName(), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(field.getName(), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(field.getName(), ((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning(ex);
}
}
// getters - methods
if (getters != null && getters.length > 0) {
List<Document> docList = new ArrayList();
for (Object key : identityKeyList) {
Document doc = luceneBuilder.createDocument();
for (Method method : getters) {
// TODO: build lucene here
Object obj = method.invoke(key);
Class<?> fieldType = method.getReturnType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(getPropertyName(method),
((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning(ex);
}
}
} catch (Exception ex) {
Logger.warning(ex);
}
}
writer.commit();
writer.close();
// TODO: support file system
// place the RamDirectory in lucene
region.put(type.getFullPath(), directory);
} catch (Exception ex) {
Logger.error("Index builder aborted.", ex);
return;
}
}
}
From source file:com.netcrest.pado.index.provider.lucene.LuceneBuilderRAMDirectory.java
License:Open Source License
@SuppressWarnings({ "rawtypes", "resource" })
public void buildTemporalData(boolean createNewDirectory) {
Cache cache = CacheFactory.getAnyInstance();
Region<String, RAMDirectory> region = cache
.getRegion(IndexMatrixUtil.getProperty(Constants.PROP_REGION_LUCENE));
try {//from ww w. j av a 2s. c o m
TemporalType[] temporalTypes = GemfireTemporalManager.getAllTemporalTypes();
for (TemporalType type : temporalTypes) {
IndexWriter writer = null;
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer);
iwc.setOpenMode(OpenMode.CREATE);
LuceneField luceneBuilder = new LuceneField();
LuceneSearch ls = LuceneSearch.getLuceneSearch(type.getFullPath());
StandardQueryParser parser = ls.createParser();
TemporalManager tm = TemporalManager.getTemporalManager(type.getFullPath());
Method[] attributeGetters = null;
boolean isIdentityKeyPrimitive = false;
try {
List identityKeyList = tm.getIdentityKeyList();
if (identityKeyList.size() == 0) {
continue;
}
RAMDirectory directory;
if (createNewDirectory) {
directory = new RAMDirectory();
} else {
directory = region.get(type.getFullPath());
if (directory == null) {
directory = new RAMDirectory();
}
}
writer = new IndexWriter(directory, iwc);
// first, find the attribute getter methods
boolean isKeyMap = false;
KeyType keyType = null;
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
Object value;
if (data instanceof GemfireTemporalData) {
value = ((GemfireTemporalData) data).getValue();
} else {
value = data;
}
isKeyMap = value instanceof KeyMap;
if (isKeyMap == false) {
attributeGetters = ReflectionHelper.getAttributeGetters(data.getClass());
} else {
keyType = ((KeyMap) value).getKeyType();
}
isIdentityKeyPrimitive = ReflectionHelper.isPrimitiveWrapper(identityKey.getClass());
break;
}
// build the numeric config
Map<String, NumericConfig> map = parser.getNumericConfigMap();
if (map == null) {
map = new HashMap<String, NumericConfig>();
parser.setNumericConfigMap(map);
}
if (isKeyMap) {
KeyType[] keyTypes = KeyTypeManager.getAllRegisteredVersions(keyType.getClass());
for (KeyType kt : keyTypes) {
Set<String> nameSet = kt.getNameSet();
for (String name : nameSet) {
if (map.containsKey(name)) {
continue;
}
KeyType kt2 = kt.getKeyType(name);
String fieldName = kt2.getName();
Class<?> fieldType = kt2.getType();
if (fieldType == Integer.class || fieldType == int.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.INT);
map.put(fieldName, config);
} else if (fieldType == Long.class || fieldType == long.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.LONG);
map.put(fieldName, config);
} else if (fieldType == Float.class || fieldType == float.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.FLOAT);
map.put(fieldName, config);
} else if (fieldType == Double.class || fieldType == double.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.DOUBLE);
map.put(fieldName, config);
} else if (fieldType == Date.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP, DATE_FORMAT,
NumericType.LONG);
map.put(fieldName, config);
}
}
}
List<Document> docList = new ArrayList<Document>();
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
KeyMap keyMap;
if (data instanceof GemfireTemporalData) {
keyMap = (KeyMap) ((GemfireTemporalData) data).getValue();
} else {
keyMap = (KeyMap) data;
}
keyType = keyMap.getKeyType();
Set<String> nameSet = keyType.getNameSet();
Document doc = luceneBuilder.createDocument();
if (isIdentityKeyPrimitive) {
doc.add(luceneBuilder.createField("IdentityKey", identityKey.toString()));
} else {
doc.add(luceneBuilder.createIdentityKeyField(identityKey));
}
for (String name : nameSet) {
Object obj = keyMap.get(name);
// obj can be null (e.g., version difference or
// app defined)
if (obj == null) {
continue;
}
KeyType kt = keyType.getKeyType(name);
Class fieldType = kt.getType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(name, obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(name, (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(name, (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(name, (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(name, (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(name, ((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning("MapLite error", ex);
}
} else {
for (Method method : attributeGetters) {
Class fieldType = method.getReturnType();
String fieldName = method.getName().substring(3);
if (fieldType == Integer.class || fieldType == int.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.INT);
map.put(fieldName, config);
} else if (fieldType == Long.class || fieldType == long.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.LONG);
map.put(fieldName, config);
} else if (fieldType == Float.class || fieldType == float.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.FLOAT);
map.put(fieldName, config);
} else if (fieldType == Double.class || fieldType == double.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.DOUBLE);
map.put(fieldName, config);
} else if (fieldType == Date.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP, DATE_FORMAT,
NumericType.LONG);
map.put(fieldName, config);
}
}
// build lucene for each attribute in the current
// (latest)
// data
if (attributeGetters != null && attributeGetters.length > 0) {
List<Document> docList = new ArrayList<Document>();
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
Document doc = luceneBuilder.createDocument();
if (isIdentityKeyPrimitive) {
doc.add(luceneBuilder.createField("IdentityKey", identityKey.toString()));
} else {
doc.add(luceneBuilder.createIdentityKeyField(identityKey));
}
for (Method method : attributeGetters) {
Object obj = method.invoke(data);
Class fieldType = method.getReturnType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(getPropertyName(method),
((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning("Non-MapLite error", ex);
}
}
}
writer.commit();
writer.close();
// TODO: support file system
// place the RamDirectory in the lucene
region.put(type.getFullPath(), directory);
} catch (Exception ex) {
Logger.warning(ex);
}
}
} catch (Exception ex) {
Logger.warning("Index builder aborted.", ex);
}
}
From source file:com.o19s.es.explore.ExplorerQueryTests.java
License:Apache License
@Before public void setupIndex() throws Exception { dir = new RAMDirectory(); try (IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER))) { for (int i = 0; i < docs.length; i++) { Document doc = new Document(); doc.add(new Field("_id", Integer.toString(i + 1), StoredField.TYPE)); doc.add(newTextField("text", docs[i], Field.Store.YES)); indexWriter.addDocument(doc); }//from ww w.j a v a 2 s .co m } reader = DirectoryReader.open(dir); searcher = new IndexSearcher(reader); }
From source file:com.o19s.solr.swan.highlight.TermVectorFun.java
License:Apache License
@Test public void testBlah() throws IOException { RAMDirectory ramDir = new RAMDirectory(); // Index some made up content IndexWriterConfig iwf = new IndexWriterConfig(Version.LUCENE_47, new StandardAnalyzer(Version.LUCENE_47)); IndexWriter writer = new IndexWriter(ramDir, iwf); FieldType ft = new FieldType(); ft.setIndexed(true);/*from w ww. ja v a 2 s . com*/ ft.setTokenized(true); ft.setStored(true); ft.setStoreTermVectorOffsets(true); ft.setStoreTermVectors(true); ft.setStoreTermVectorPositions(true); ft.freeze(); for (int i = 0; i < DOCS.length; i++) { Document doc = new Document(); StringField id = new StringField("id", "doc_" + i, StringField.Store.YES); doc.add(id); // Store both position and offset information Field text = new Field("content", DOCS[i], ft); // Field.Index.ANALYZED, // Field.TermVector.WITH_POSITIONS_OFFSETS); doc.add(text); writer.addDocument(doc); } //writer.close(); // Get a searcher AtomicReader dr = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(writer, true)); IndexSearcher searcher = new IndexSearcher(dr); // Do a search using SpanQuery SpanTermQuery fleeceQ = new SpanTermQuery(new Term("content", "fleece")); TopDocs results = searcher.search(fleeceQ, 10); for (int i = 0; i < results.scoreDocs.length; i++) { ScoreDoc scoreDoc = results.scoreDocs[i]; System.out.println("Score Doc: " + scoreDoc); } IndexReader reader = searcher.getIndexReader(); Bits acceptDocs = null; Map<Term, TermContext> termContexts = new HashMap<Term, TermContext>(); Spans spans = fleeceQ.getSpans(dr.getContext(), acceptDocs, termContexts); while (spans.next()) { System.out.println("Doc: " + spans.doc() + " Start: " + spans.start() + " End: " + spans.end()); DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor("content"); reader.document(spans.doc(), visitor); Terms terms = reader.getTermVector(spans.doc(), "content"); TermsEnum tenum = terms.iterator(null); // AttributeSource as = tenum.attributes(); while (tenum.next() != null) { System.out.println(tenum.term().utf8ToString()); } for (long pos = 0L; pos < spans.end(); pos++) { // tenum.next(); // if (tenum.ord()<pos) continue; // System.out.println(tenum.term()); // } reader.document(spans.doc(), visitor); // String[] values = visitor.getDocument().getValues("content"); // List<String> a = new ArrayList<String>(); // // build up the window // tvm.start = spans.start() - window; // tvm.end = spans.end() + window; // reader.getTermFreqVector(spans.doc(), "content", tvm); // for (WindowEntry entry : tvm.entries.values()) { // System.out.println("Entry: " + entry); // } // // clear out the entries for the next round // tvm.entries.clear(); } }
From source file:com.orientechnologies.lucene.engine.OLuceneFacetManager.java
License:Apache License
private Directory getTaxDirectory(ODatabaseDocumentInternal database) throws IOException { final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage() .getUnderlying();/*from www . j a v a 2 s . c o m*/ if (storageLocalAbstract instanceof OLocalPaginatedStorage) { String pathname = getIndexFacetPath((OLocalPaginatedStorage) storageLocalAbstract); return NIOFSDirectory.open(new File(pathname).toPath()); } return new RAMDirectory(); }
From source file:com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract.java
License:Apache License
@Override public OLuceneTxChanges buildTxChanges() throws IOException { if (isCollectionDelete()) { return new OLuceneTxChangesMultiRid(this, createIndexWriter(new RAMDirectory()), createIndexWriter(new RAMDirectory())); } else {//from ww w . jav a 2 s. co m return new OLuceneTxChangesSingleRid(this, createIndexWriter(new RAMDirectory()), createIndexWriter(new RAMDirectory())); } }
From source file:com.orientechnologies.lucene.engine.OLuceneStorage.java
License:Apache License
private void reOpen() throws IOException { if (mgrWriter != null) { OLogManager.instance().info(this, "index storage is open don't reopen"); return;/* w w w. j a v a2 s.c om*/ } ODatabaseDocumentInternal database = ODatabaseRecordThreadLocal.INSTANCE.get(); final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage() .getUnderlying(); Directory dir = null; if (storageLocalAbstract instanceof OLocalPaginatedStorage) { String pathname = getIndexPath((OLocalPaginatedStorage) storageLocalAbstract); OLogManager.instance().info(this, "Opening NIOFS Lucene db=%s, path=%s", database.getName(), pathname); dir = NIOFSDirectory.open(new File(pathname).toPath()); } else { OLogManager.instance().info(this, "Opening RAM Lucene index db=%s", database.getName()); dir = new RAMDirectory(); } final IndexWriter indexWriter = createIndexWriter(dir); mgrWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, null); if (nrt != null) { nrt.close(); } nrt = new ControlledRealTimeReopenThread(mgrWriter, searcherManager, 60.00, 0.1); nrt.setDaemon(true); nrt.start(); flush(); OLogManager.instance().info(this, "REOPEN DONE"); }
From source file:com.orientechnologies.lucene.manager.OLuceneFacetManager.java
License:Apache License
private Directory getTaxDirectory(ODatabaseDocumentInternal database) throws IOException { Directory dir = null;// w w w .j a v a 2 s . c o m final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage() .getUnderlying(); if (storageLocalAbstract instanceof OLocalPaginatedStorage) { String pathname = getIndexFacetPath((OLocalPaginatedStorage) storageLocalAbstract); dir = NIOFSDirectory.open(new File(pathname)); } else { dir = new RAMDirectory(); } return dir; }
From source file:com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract.java
License:Apache License
private void reOpen(final ODocument metadata) throws IOException { ODatabaseDocumentInternal database = getDatabase(); final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage() .getUnderlying();// w ww . ja v a 2s.c o m Directory dir = null; if (storageLocalAbstract instanceof OLocalPaginatedStorage) { String pathname = getIndexPath((OLocalPaginatedStorage) storageLocalAbstract); OLogManager.instance().debug(this, "Opening NIOFS Lucene db=%s, path=%s", database.getName(), pathname); dir = NIOFSDirectory.open(new File(pathname)); } else { OLogManager.instance().debug(this, "Opening RAM Lucene index db=%s", database.getName()); dir = new RAMDirectory(); } final IndexWriter indexWriter = createIndexWriter(dir, metadata); mgrWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, null); if (nrt != null) { nrt.close(); } nrt = new ControlledRealTimeReopenThread(mgrWriter, searcherManager, 60.00, 0.1); nrt.setDaemon(true); nrt.start(); flush(); }