List of usage examples for org.apache.lucene.index IndexWriter addDocuments
public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException
From source file:IndexTaxis.java
License:Apache License
/** Index all documents contained in one chunk */ static void indexOneChunk(String[] fields, Chunk chunk, IndexWriter w, AtomicInteger docCounter, AtomicLong bytesCounter) throws IOException { Document doc = new Document(); byte[] bytes = chunk.bytes; if (bytes[bytes.length - 1] != NEWLINE) { throw new AssertionError(); }//w w w . j a v a 2s . c om w.addDocuments(new Iterable<Document>() { @Override public Iterator<Document> iterator() { return new Iterator<Document>() { private int i; private Document nextDoc; private boolean nextSet; private int lastLineStart; private int chunkDocCount; @Override public boolean hasNext() { if (nextSet == false) { setNextDoc(); nextSet = true; } return nextDoc != null; } @Override public Document next() { assert nextSet; nextSet = false; Document result = nextDoc; nextDoc = null; return result; } private void setNextDoc() { Document doc = new Document(); int fieldUpto = 0; int lastFieldStart = i; for (; i < bytes.length; i++) { byte b = bytes[i]; if (b == NEWLINE || b == COMMA) { if (i > lastFieldStart) { String s = new String(bytes, lastFieldStart, i - lastFieldStart, StandardCharsets.UTF_8); addOneField(doc, fields[fieldUpto], s); } if (b == NEWLINE) { if (fieldUpto != fields.length - 1) { throw new AssertionError("fieldUpto=" + fieldUpto + " vs fields.length-1=" + (fields.length - 1)); } chunkDocCount++; this.nextDoc = doc; int x = docCounter.incrementAndGet(); long y = bytesCounter.addAndGet((i + 1) - lastLineStart); if (x % 100000 == 0) { double sec = (System.nanoTime() - startNS) / 1000000000.0; System.out.println(String.format(Locale.ROOT, "%.1f sec: %d docs; %.1f docs/sec; %.1f MB/sec", sec, x, x / sec, (y / 1024. / 1024.) / sec)); } fieldUpto = 0; i++; lastLineStart = i; return; } else { fieldUpto++; } lastFieldStart = i + 1; } } // System.out.println("chunk doc count: " + chunkDocCount); } }; } }); }
From source file:collene.Freedb.java
License:Apache License
public static void BuildIndex(Directory directory) throws Exception { String freedbPath = "/Users/gdusbabek/Downloads/freedb-complete-20140701.tar.bz2"; if (directory == null) { System.out.println("Need to specify: { memory | file | cassandra }. Did you misspell something?"); System.exit(-1);/*w ww . ja v a 2s. co m*/ } FreeDbReader reader = new FreeDbReader(new File(freedbPath), 50000); reader.start(); long indexStart = System.currentTimeMillis(); Collection<Document> documents = new ArrayList<Document>(BATCH_SIZE); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); IndexWriter writer = new IndexWriter(directory, config); // stop after this many documents. final int maxDocuments = 400000; //Integer.MAX_VALUE; FreeDbEntry entry = reader.next(); int count = 0; while (entry != null && count < maxDocuments) { Document doc = new Document(); String any = entry.toString(); doc.add(new Field("any", any, TextField.TYPE_STORED)); doc.add(new Field("artist", entry.getArtist(), TextField.TYPE_NOT_STORED)); doc.add(new Field("album", entry.getAlbum(), TextField.TYPE_NOT_STORED)); doc.add(new Field("title", entry.getTitle(), TextField.TYPE_NOT_STORED)); doc.add(new Field("genre", entry.getGenre(), TextField.TYPE_NOT_STORED)); doc.add(new Field("year", entry.getYear(), TextField.TYPE_NOT_STORED)); for (int i = 0; i < entry.getTrackCount(); i++) { doc.add(new Field("track", entry.getTrack(i), TextField.TYPE_STORED)); } documents.add(doc); if (VERBOSE) { out.println(any); } if (documents.size() == BATCH_SIZE) { //out.println(String.format("Adding batch at count %d", count)); writer.addDocuments(documents); //out.println("done"); documents.clear(); } count += 1; if (count >= MAX_ENTRIES) { // done indexing. break; } entry = reader.next(); if (count % 100000 == 0) { out.println(String.format("Indexed %d documents", count)); // do a quick morrissey search for fun. // IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(ColDirectory.open( // new CassandraIO(8192, "collene", "cindex").start("127.0.0.1:9042"), // new CassandraIO(8192, "collene", "cmeta").start("127.0.0.1:9042"), // new CassandraIO(8192, "collene", "clock").start("127.0.0.1:9042") // ))); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); QueryParser parser = new QueryParser(Version.LUCENE_4_9, "any", analyzer); long searchStart = System.currentTimeMillis(); Query query = parser.parse("morrissey"); TopDocs docs = searcher.search(query, 10); long searchEnd = System.currentTimeMillis(); out.println(String.format("%s %d total hits in %d", directory.getClass().getSimpleName(), docs.totalHits, searchEnd - searchStart)); for (ScoreDoc d : docs.scoreDocs) { out.println(String.format("%d %.2f %d", d.doc, d.score, d.shardIndex)); } } } if (documents.size() > 0) { out.println(String.format("Adding batch at count %d", count)); writer.addDocuments(documents); out.println("done"); documents.clear(); // do a quick morrissey search for fun. IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); QueryParser parser = new QueryParser(Version.LUCENE_4_9, "any", analyzer); long searchStart = System.currentTimeMillis(); Query query = parser.parse("morrissey"); TopDocs docs = searcher.search(query, 10); long searchEnd = System.currentTimeMillis(); out.println(String.format("%s %d total hits in %d", directory.getClass().getSimpleName(), docs.totalHits, searchEnd - searchStart)); for (ScoreDoc d : docs.scoreDocs) { out.println(String.format("%d %.2f %d", d.doc, d.score, d.shardIndex)); } } long indexTime = System.currentTimeMillis() - indexStart; out.println(String.format("Indexed %d things in %d ms (%s)", count, indexTime, directory.toString())); // long startMerge = System.currentTimeMillis(); // writer.forceMerge(1, true); // long endMerge = System.currentTimeMillis(); // out.println(String.format("merge took %d ms", endMerge-startMerge)); out.println("I think these are the files:"); for (String s : directory.listAll()) { out.println(s); } writer.close(true); directory.close(); }
From source file:collene.TestIndexing.java
License:Apache License
@Test public void test() throws IOException, ParseException { Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9); // write it out. IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(directory, config); for (int i = 0; i < 100; i++) { Collection<Document> documents = new ArrayList<Document>(); Document doc = new Document(); doc.add(new Field("key", "aaa_" + i, TextField.TYPE_STORED)); doc.add(new Field("not", "notaaa", TextField.TYPE_NOT_STORED)); doc.add(new Field("meta", "aaa_meta_aaa_" + i, TextField.TYPE_STORED)); documents.add(doc);/*from w w w . j av a2 s . c o m*/ writer.addDocuments(documents); writer.commit(); writer.forceMerge(1); writer.forceMergeDeletes(true); } // now read it back. IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); QueryParser parser = new QueryParser(Version.LUCENE_4_9, "key", analyzer); Query query = parser.parse("aaa_4"); TopDocs docs = searcher.search(query, 1); int idToDelete = docs.scoreDocs[0].doc; Assert.assertTrue(docs.totalHits > 0); query = parser.parse("fersoius"); docs = searcher.search(query, 1); Assert.assertFalse(docs.totalHits > 0); // delete that document. DirectoryReader reader = DirectoryReader.open(writer, true); writer.tryDeleteDocument(reader, idToDelete); reader.close(); writer.close(); // list files Set<String> files = new HashSet<String>(); System.out.println("Listing files for " + directory.toString()); for (String file : directory.listAll()) { files.add(file); System.out.println(" " + file); } if (strictFileChecking) { System.out.println("String file checking..."); Sets.SetView<String> difference = Sets.difference(expectedFiles, files); Assert.assertEquals(Joiner.on(",").join(difference), 0, difference.size()); } reader = DirectoryReader.open(directory); searcher = new IndexSearcher(reader); query = parser.parse("aaa_4"); docs = searcher.search(query, 1); reader.close(); Assert.assertFalse(docs.totalHits > 0); directory.close(); }
From source file:collene.TestShakespeare.java
License:Apache License
@Test public void rest() throws IOException, ParseException { File shakespeareDir = new File("src/test/resources/shakespeare"); File[] files = shakespeareDir.listFiles(new FileFilter() { @Override//from w w w.ja v a2s .co m public boolean accept(File pathname) { return !pathname.isHidden(); } }); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); IndexWriter writer = new IndexWriter(directory, config); long startIndexTime = System.currentTimeMillis(); final int flushLines = 200; int totalLines = 0; Collection<Document> documents = new ArrayList<Document>(); for (File f : files) { String play = f.getName(); int lineNumber = 1; BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(f))); String line = reader.readLine(); while (line != null) { // index it. Document doc = new Document(); doc.add(new NumericDocValuesField("line", lineNumber)); doc.add(new Field("play", play, TextField.TYPE_STORED)); doc.add(new Field("content", line, TextField.TYPE_STORED)); documents.add(doc); totalLines += 1; if (totalLines % flushLines == 0) { writer.addDocuments(documents); documents.clear(); } lineNumber += 1; line = reader.readLine(); } reader.close(); } if (documents.size() > 0) { writer.addDocuments(documents); } long endIndexTime = System.currentTimeMillis(); System.out.println( String.format("Index for %s took %d ms", directory.toString(), endIndexTime - startIndexTime)); //System.out.println(String.format("%s committed", directory.getClass().getSimpleName())); // writer.forceMerge(1); // System.out.println(String.format("%s merged", directory.getClass().getSimpleName())); // let's search! IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false)); QueryParser parser = new QueryParser(Version.LUCENE_4_9, "content", analyzer); String[] queryTerms = new String[] { "trumpet" }; for (String term : queryTerms) { long searchStart = System.currentTimeMillis(); Query query = parser.parse(term); TopDocs docs = searcher.search(query, 10); long searchEnd = System.currentTimeMillis(); System.out.println(String.format("%s %d total hits in %d", directory.getClass().getSimpleName(), docs.totalHits, searchEnd - searchStart)); for (ScoreDoc doc : docs.scoreDocs) { System.out.println(String.format("%d %.2f %d", doc.doc, doc.score, doc.shardIndex)); } } writer.close(true); //System.out.println(String.format("%s closed", directory.getClass().getSimpleName())); System.out.println("I think these are the files:"); for (String s : directory.listAll()) { System.out.println(s); } directory.close(); }
From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java
License:Apache License
/** * ??/* w ww.j av a 2 s .c om*/ * @param module ? * @param docs * @param isNew * @param delTerm del * @throws IOException */ @SuppressWarnings("deprecation") public static void adds(String module, List<Document> docs, boolean isNew, Term delTerm) throws IOException { long beginTime = System.currentTimeMillis(); IndexWriter writer = null; try { Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module))); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer); iwc.setMaxBufferedDocs(100); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); // iwc.setRAMBufferSizeMB(256.0);// ? writer = new IndexWriter(dir, iwc); if (isNew) { writer.addDocuments(docs); } else { writer.updateDocuments(delTerm, docs); } writer.commit(); } finally { long endTime = System.currentTimeMillis(); logger.debug("adds consume " + (endTime - beginTime) + " milliseconds."); if (writer != null) writer.close(); } }
From source file:com.netcrest.pado.index.gemfire.lucene.TemporalLuceneDynamicIndexing.java
License:Open Source License
/** * Builds Lucene indexes for the specified temporal events. * /*from w w w.ja va 2 s.c om*/ * @param parser * Lucene parser * @param events * Temporal events * @param writer * Index writer */ private void buildTemporalEntries(StandardQueryParser parser, List<EntryEvent<ITemporalKey, ITemporalData>> events, IndexWriter writer) { LuceneBuilder luceneBuilder = LuceneBuilder.getLuceneBuilder(); boolean isKeyMap = false; KeyType keyType = null; Set<Object> keySet = null; Object firstDataObject = null; Method[] attributeGetters = null; boolean isIdentityKeyPrimitive = false; for (EntryEvent<ITemporalKey, ITemporalData> event : events) { ITemporalKey tk = event.getKey(); ITemporalData data = event.getNewValue(); if (data instanceof GemfireTemporalData) { firstDataObject = ((GemfireTemporalData) data).getValue(); } else { firstDataObject = data; } isKeyMap = firstDataObject instanceof KeyMap; if (isKeyMap == false) { if (firstDataObject instanceof Map) { keySet = ((Map) firstDataObject).keySet(); } else { attributeGetters = ReflectionHelper.getAttributeGetters(data.getClass()); } } else { keyType = ((KeyMap) firstDataObject).getKeyType(); if (keyType == null) { keySet = ((Map) firstDataObject).keySet(); } } Object identityKey = tk.getIdentityKey(); isIdentityKeyPrimitive = ReflectionHelper.isPrimitiveWrapper(identityKey.getClass()); break; } LuceneField luceneField = new LuceneField(); SimpleDateFormat format = (SimpleDateFormat) DateTool.Resolution.DAY.format.clone(); List<Document> docList = new ArrayList<Document>(); try { if (keyType != null) { for (EntryEvent<ITemporalKey, ITemporalData> event : events) { ITemporalKey tk = event.getKey(); ITemporalData data = event.getNewValue(); KeyMap keyMap; if (data instanceof GemfireTemporalData) { keyMap = (KeyMap) ((GemfireTemporalData) data).getValue(); } else { keyMap = (KeyMap) data; } keyType = keyMap.getKeyType(); Set<String> nameSet = keyType.getNameSet(); // TODO: See if we can support binary types // createDoc(); Document doc = luceneBuilder.createKeyMapDocument(parser, writer, tk, data, -1, luceneField, keyType, keyMap, nameSet, isIdentityKeyPrimitive, true, format); docList.add(doc); } } else if (keySet != null) { for (EntryEvent<ITemporalKey, ITemporalData> event : events) { ITemporalKey tk = event.getKey(); ITemporalData data = event.getNewValue(); Map dataMap; if (data instanceof GemfireTemporalData) { dataMap = (Map) ((GemfireTemporalData) data).getValue(); } else { dataMap = (Map) data; } // TODO: See if we can support binary types // createDoc(); Document doc = luceneBuilder.createMapDocument(parser, writer, tk, data, luceneField, dataMap, keySet, isIdentityKeyPrimitive, format); docList.add(doc); } } else { if (attributeGetters != null && attributeGetters.length > 0) { for (EntryEvent<ITemporalKey, ITemporalData> event : events) { ITemporalKey tk = event.getKey(); ITemporalData data = event.getNewValue(); Document doc = luceneBuilder.createPojoDocument(parser, writer, tk, data, -1l, luceneField, attributeGetters, isIdentityKeyPrimitive, true/* isNew */, format); docList.add(doc); } } } } catch (Exception ex) { Logger.error(ex); } try { writer.addDocuments(docList); } catch (Exception ex) { Logger.error(ex); } }
From source file:com.netcrest.pado.index.provider.lucene.LuceneBuilderRAMDirectory.java
License:Open Source License
@SuppressWarnings({ "unchecked", "rawtypes", "resource" })
public void buildTemporalKeys(boolean createNewDirectory) {
Cache cache = CacheFactory.getAnyInstance();
Region<String, RAMDirectory> region = cache
.getRegion(IndexMatrixUtil.getProperty(Constants.PROP_REGION_LUCENE));
TemporalType[] temporalTypes = GemfireTemporalManager.getAllTemporalTypes();
for (TemporalType type : temporalTypes) {
IndexWriter writer = null;
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer);
iwc.setOpenMode(OpenMode.CREATE);
LuceneField luceneBuilder = new LuceneField();
LuceneSearch ls = LuceneSearch.getLuceneSearch(type.getFullPath());
StandardQueryParser parser = ls.createParser();
TemporalManager tm = TemporalManager.getTemporalManager(type.getFullPath());
try {/* ww w . j a v a 2 s. c o m*/
List<?> identityKeyList = tm.getIdentityKeyList();
if (identityKeyList.size() == 0) {
continue;
}
RAMDirectory directory;
if (createNewDirectory) {
directory = new RAMDirectory();
} else {
directory = region.get(type.getFullPath());
if (directory == null) {
directory = new RAMDirectory();
}
}
writer = new IndexWriter(directory, iwc);
// determine the identity key type, public fields and getters
Field fields[] = null;
Method getters[] = null;
Class keyType = null;
for (Object key : identityKeyList) {
if (ReflectionHelper.isPrimitiveWrapper(key.getClass())) {
fields = null;
getters = null;
keyType = key.getClass();
} else {
fields = ReflectionHelper.getPublicFields(key.getClass());
getters = ReflectionHelper.getPublicGetters(key.getClass());
}
break;
}
if (keyType != null) {
configNumericType(parser, "IdentityKey", keyType);
// primitive
List<Document> docList = new ArrayList();
if (keyType == String.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", key.toString()));
docList.add(doc);
}
} else if (keyType == Integer.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Integer) key));
docList.add(doc);
}
} else if (keyType == Long.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Long) key));
docList.add(doc);
}
} else if (keyType == Float.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Float) key));
docList.add(doc);
}
} else if (keyType == Double.class) {
for (Object key : identityKeyList) {
// TODO: do lucene here
Document doc = luceneBuilder.createDocument();
doc.add(luceneBuilder.createField("IdentityKey", (Double) key));
docList.add(doc);
}
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning(ex);
}
} else {
try {
// fields
if (fields != null && fields.length > 0) {
// configure numeric types
for (Field field : fields) {
configNumericType(parser, field.getName(), field.getType());
}
List<Document> docList = new ArrayList();
for (Object key : identityKeyList) {
Document doc = luceneBuilder.createDocument();
for (Field field : fields) {
Object obj = field.get(key);
Class fieldType = field.getType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(field.getName(), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(field.getName(), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(field.getName(), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(field.getName(), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(field.getName(), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(field.getName(), ((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning(ex);
}
}
// getters - methods
if (getters != null && getters.length > 0) {
List<Document> docList = new ArrayList();
for (Object key : identityKeyList) {
Document doc = luceneBuilder.createDocument();
for (Method method : getters) {
// TODO: build lucene here
Object obj = method.invoke(key);
Class<?> fieldType = method.getReturnType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(getPropertyName(method),
((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning(ex);
}
}
} catch (Exception ex) {
Logger.warning(ex);
}
}
writer.commit();
writer.close();
// TODO: support file system
// place the RamDirectory in lucene
region.put(type.getFullPath(), directory);
} catch (Exception ex) {
Logger.error("Index builder aborted.", ex);
return;
}
}
}
From source file:com.netcrest.pado.index.provider.lucene.LuceneBuilderRAMDirectory.java
License:Open Source License
@SuppressWarnings({ "rawtypes", "resource" })
public void buildTemporalData(boolean createNewDirectory) {
Cache cache = CacheFactory.getAnyInstance();
Region<String, RAMDirectory> region = cache
.getRegion(IndexMatrixUtil.getProperty(Constants.PROP_REGION_LUCENE));
try {//from w w w . j a v a2 s . c o m
TemporalType[] temporalTypes = GemfireTemporalManager.getAllTemporalTypes();
for (TemporalType type : temporalTypes) {
IndexWriter writer = null;
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer);
iwc.setOpenMode(OpenMode.CREATE);
LuceneField luceneBuilder = new LuceneField();
LuceneSearch ls = LuceneSearch.getLuceneSearch(type.getFullPath());
StandardQueryParser parser = ls.createParser();
TemporalManager tm = TemporalManager.getTemporalManager(type.getFullPath());
Method[] attributeGetters = null;
boolean isIdentityKeyPrimitive = false;
try {
List identityKeyList = tm.getIdentityKeyList();
if (identityKeyList.size() == 0) {
continue;
}
RAMDirectory directory;
if (createNewDirectory) {
directory = new RAMDirectory();
} else {
directory = region.get(type.getFullPath());
if (directory == null) {
directory = new RAMDirectory();
}
}
writer = new IndexWriter(directory, iwc);
// first, find the attribute getter methods
boolean isKeyMap = false;
KeyType keyType = null;
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
Object value;
if (data instanceof GemfireTemporalData) {
value = ((GemfireTemporalData) data).getValue();
} else {
value = data;
}
isKeyMap = value instanceof KeyMap;
if (isKeyMap == false) {
attributeGetters = ReflectionHelper.getAttributeGetters(data.getClass());
} else {
keyType = ((KeyMap) value).getKeyType();
}
isIdentityKeyPrimitive = ReflectionHelper.isPrimitiveWrapper(identityKey.getClass());
break;
}
// build the numeric config
Map<String, NumericConfig> map = parser.getNumericConfigMap();
if (map == null) {
map = new HashMap<String, NumericConfig>();
parser.setNumericConfigMap(map);
}
if (isKeyMap) {
KeyType[] keyTypes = KeyTypeManager.getAllRegisteredVersions(keyType.getClass());
for (KeyType kt : keyTypes) {
Set<String> nameSet = kt.getNameSet();
for (String name : nameSet) {
if (map.containsKey(name)) {
continue;
}
KeyType kt2 = kt.getKeyType(name);
String fieldName = kt2.getName();
Class<?> fieldType = kt2.getType();
if (fieldType == Integer.class || fieldType == int.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.INT);
map.put(fieldName, config);
} else if (fieldType == Long.class || fieldType == long.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.LONG);
map.put(fieldName, config);
} else if (fieldType == Float.class || fieldType == float.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.FLOAT);
map.put(fieldName, config);
} else if (fieldType == Double.class || fieldType == double.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.DOUBLE);
map.put(fieldName, config);
} else if (fieldType == Date.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP, DATE_FORMAT,
NumericType.LONG);
map.put(fieldName, config);
}
}
}
List<Document> docList = new ArrayList<Document>();
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
KeyMap keyMap;
if (data instanceof GemfireTemporalData) {
keyMap = (KeyMap) ((GemfireTemporalData) data).getValue();
} else {
keyMap = (KeyMap) data;
}
keyType = keyMap.getKeyType();
Set<String> nameSet = keyType.getNameSet();
Document doc = luceneBuilder.createDocument();
if (isIdentityKeyPrimitive) {
doc.add(luceneBuilder.createField("IdentityKey", identityKey.toString()));
} else {
doc.add(luceneBuilder.createIdentityKeyField(identityKey));
}
for (String name : nameSet) {
Object obj = keyMap.get(name);
// obj can be null (e.g., version difference or
// app defined)
if (obj == null) {
continue;
}
KeyType kt = keyType.getKeyType(name);
Class fieldType = kt.getType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(name, obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(name, (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(name, (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(name, (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(name, (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(name, ((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning("MapLite error", ex);
}
} else {
for (Method method : attributeGetters) {
Class fieldType = method.getReturnType();
String fieldName = method.getName().substring(3);
if (fieldType == Integer.class || fieldType == int.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.INT);
map.put(fieldName, config);
} else if (fieldType == Long.class || fieldType == long.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.LONG);
map.put(fieldName, config);
} else if (fieldType == Float.class || fieldType == float.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.FLOAT);
map.put(fieldName, config);
} else if (fieldType == Double.class || fieldType == double.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.DOUBLE);
map.put(fieldName, config);
} else if (fieldType == Date.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP, DATE_FORMAT,
NumericType.LONG);
map.put(fieldName, config);
}
}
// build lucene for each attribute in the current
// (latest)
// data
if (attributeGetters != null && attributeGetters.length > 0) {
List<Document> docList = new ArrayList<Document>();
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
Document doc = luceneBuilder.createDocument();
if (isIdentityKeyPrimitive) {
doc.add(luceneBuilder.createField("IdentityKey", identityKey.toString()));
} else {
doc.add(luceneBuilder.createIdentityKeyField(identityKey));
}
for (Method method : attributeGetters) {
Object obj = method.invoke(data);
Class fieldType = method.getReturnType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(getPropertyName(method),
((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning("Non-MapLite error", ex);
}
}
}
writer.commit();
writer.close();
// TODO: support file system
// place the RamDirectory in the lucene
region.put(type.getFullPath(), directory);
} catch (Exception ex) {
Logger.warning(ex);
}
}
} catch (Exception ex) {
Logger.warning("Index builder aborted.", ex);
}
}
From source file:org.apache.blur.analysis.type.AclDiscoverFieldTypeDefinitionTest.java
License:Apache License
@Before public void setup() throws IOException { _fieldManager = getFieldManager(new NoStopWordStandardAnalyzer()); setupFieldManager(_fieldManager);// w w w . j a v a 2 s . c o m List<List<Field>> docs = new ArrayList<List<Field>>(); { Record record = new Record(); record.setFamily(FAM); record.setRecordId("1234"); record.addToColumns(new Column("string", "value")); record.addToColumns(new Column("discover", "a&b")); List<Field> fields = _fieldManager.getFields("1234", record); fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); docs.add(fields); } { Record record = new Record(); record.setFamily(FAM); record.setRecordId("5678"); record.addToColumns(new Column("string", "value")); record.addToColumns(new Column("discover", "a&c")); docs.add(_fieldManager.getFields("1234", record)); } IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, _fieldManager.getAnalyzerForIndex()); IndexWriter writer = new IndexWriter(_dir, conf); writer.addDocuments(docs); writer.close(); }
From source file:org.apache.blur.analysis.type.AclReadFieldTypeDefinitionTest.java
License:Apache License
@Before public void setup() throws IOException { _fieldManager = getFieldManager(new NoStopWordStandardAnalyzer()); setupFieldManager(_fieldManager);/*from www . j av a 2s.co m*/ List<List<Field>> docs = new ArrayList<List<Field>>(); { Record record = new Record(); record.setFamily(FAM); record.setRecordId("1234"); record.addToColumns(new Column("string", "value")); record.addToColumns(new Column("read", "a&b")); List<Field> fields = _fieldManager.getFields("1234", record); fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); docs.add(fields); } { Record record = new Record(); record.setFamily(FAM); record.setRecordId("5678"); record.addToColumns(new Column("string", "value")); record.addToColumns(new Column("read", "a&c")); docs.add(_fieldManager.getFields("1234", record)); } IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, _fieldManager.getAnalyzerForIndex()); IndexWriter writer = new IndexWriter(_dir, conf); writer.addDocuments(docs); writer.close(); }