List of usage examples for org.apache.lucene.index IndexWriter commit
@Override public final long commit() throws IOException
Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.
From source file:com.netcrest.pado.index.provider.lucene.LuceneBuilderRAMDirectory.java
License:Open Source License
@SuppressWarnings({ "rawtypes", "resource" })
public void buildTemporalData(boolean createNewDirectory) {
Cache cache = CacheFactory.getAnyInstance();
Region<String, RAMDirectory> region = cache
.getRegion(IndexMatrixUtil.getProperty(Constants.PROP_REGION_LUCENE));
try {/*from w ww . j a v a 2 s . c o m*/
TemporalType[] temporalTypes = GemfireTemporalManager.getAllTemporalTypes();
for (TemporalType type : temporalTypes) {
IndexWriter writer = null;
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer);
iwc.setOpenMode(OpenMode.CREATE);
LuceneField luceneBuilder = new LuceneField();
LuceneSearch ls = LuceneSearch.getLuceneSearch(type.getFullPath());
StandardQueryParser parser = ls.createParser();
TemporalManager tm = TemporalManager.getTemporalManager(type.getFullPath());
Method[] attributeGetters = null;
boolean isIdentityKeyPrimitive = false;
try {
List identityKeyList = tm.getIdentityKeyList();
if (identityKeyList.size() == 0) {
continue;
}
RAMDirectory directory;
if (createNewDirectory) {
directory = new RAMDirectory();
} else {
directory = region.get(type.getFullPath());
if (directory == null) {
directory = new RAMDirectory();
}
}
writer = new IndexWriter(directory, iwc);
// first, find the attribute getter methods
boolean isKeyMap = false;
KeyType keyType = null;
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
Object value;
if (data instanceof GemfireTemporalData) {
value = ((GemfireTemporalData) data).getValue();
} else {
value = data;
}
isKeyMap = value instanceof KeyMap;
if (isKeyMap == false) {
attributeGetters = ReflectionHelper.getAttributeGetters(data.getClass());
} else {
keyType = ((KeyMap) value).getKeyType();
}
isIdentityKeyPrimitive = ReflectionHelper.isPrimitiveWrapper(identityKey.getClass());
break;
}
// build the numeric config
Map<String, NumericConfig> map = parser.getNumericConfigMap();
if (map == null) {
map = new HashMap<String, NumericConfig>();
parser.setNumericConfigMap(map);
}
if (isKeyMap) {
KeyType[] keyTypes = KeyTypeManager.getAllRegisteredVersions(keyType.getClass());
for (KeyType kt : keyTypes) {
Set<String> nameSet = kt.getNameSet();
for (String name : nameSet) {
if (map.containsKey(name)) {
continue;
}
KeyType kt2 = kt.getKeyType(name);
String fieldName = kt2.getName();
Class<?> fieldType = kt2.getType();
if (fieldType == Integer.class || fieldType == int.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.INT);
map.put(fieldName, config);
} else if (fieldType == Long.class || fieldType == long.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.LONG);
map.put(fieldName, config);
} else if (fieldType == Float.class || fieldType == float.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.FLOAT);
map.put(fieldName, config);
} else if (fieldType == Double.class || fieldType == double.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.DOUBLE);
map.put(fieldName, config);
} else if (fieldType == Date.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP, DATE_FORMAT,
NumericType.LONG);
map.put(fieldName, config);
}
}
}
List<Document> docList = new ArrayList<Document>();
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
KeyMap keyMap;
if (data instanceof GemfireTemporalData) {
keyMap = (KeyMap) ((GemfireTemporalData) data).getValue();
} else {
keyMap = (KeyMap) data;
}
keyType = keyMap.getKeyType();
Set<String> nameSet = keyType.getNameSet();
Document doc = luceneBuilder.createDocument();
if (isIdentityKeyPrimitive) {
doc.add(luceneBuilder.createField("IdentityKey", identityKey.toString()));
} else {
doc.add(luceneBuilder.createIdentityKeyField(identityKey));
}
for (String name : nameSet) {
Object obj = keyMap.get(name);
// obj can be null (e.g., version difference or
// app defined)
if (obj == null) {
continue;
}
KeyType kt = keyType.getKeyType(name);
Class fieldType = kt.getType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(name, obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(name, (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(name, (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(name, (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(name, (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(name, ((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning("MapLite error", ex);
}
} else {
for (Method method : attributeGetters) {
Class fieldType = method.getReturnType();
String fieldName = method.getName().substring(3);
if (fieldType == Integer.class || fieldType == int.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.INT);
map.put(fieldName, config);
} else if (fieldType == Long.class || fieldType == long.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.LONG);
map.put(fieldName, config);
} else if (fieldType == Float.class || fieldType == float.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.FLOAT);
map.put(fieldName, config);
} else if (fieldType == Double.class || fieldType == double.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP,
NumberFormat.getNumberInstance(), NumericType.DOUBLE);
map.put(fieldName, config);
} else if (fieldType == Date.class) {
NumericConfig config = new NumericConfig(PRECISION_STEP, DATE_FORMAT,
NumericType.LONG);
map.put(fieldName, config);
}
}
// build lucene for each attribute in the current
// (latest)
// data
if (attributeGetters != null && attributeGetters.length > 0) {
List<Document> docList = new ArrayList<Document>();
for (Object identityKey : identityKeyList) {
TemporalEntry entry = tm.getLastEntry(identityKey);
ITemporalData data = entry.getTemporalData();
Document doc = luceneBuilder.createDocument();
if (isIdentityKeyPrimitive) {
doc.add(luceneBuilder.createField("IdentityKey", identityKey.toString()));
} else {
doc.add(luceneBuilder.createIdentityKeyField(identityKey));
}
for (Method method : attributeGetters) {
Object obj = method.invoke(data);
Class fieldType = method.getReturnType();
if (fieldType == String.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), obj.toString()));
} else if (fieldType == Integer.class || fieldType == int.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Integer) obj));
} else if (fieldType == Long.class || fieldType == long.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Long) obj));
} else if (fieldType == Float.class || fieldType == float.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Float) obj));
} else if (fieldType == Double.class || fieldType == double.class) {
doc.add(luceneBuilder.createField(getPropertyName(method), (Double) obj));
} else if (fieldType == Date.class) {
doc.add(luceneBuilder.createField(getPropertyName(method),
((Date) obj).getTime()));
}
}
docList.add(doc);
}
try {
writer.addDocuments(docList);
} catch (Exception ex) {
Logger.warning("Non-MapLite error", ex);
}
}
}
writer.commit();
writer.close();
// TODO: support file system
// place the RamDirectory in the lucene
region.put(type.getFullPath(), directory);
} catch (Exception ex) {
Logger.warning(ex);
}
}
} catch (Exception ex) {
Logger.warning("Index builder aborted.", ex);
}
}
From source file:com.nuvolect.deepdive.lucene.Index.java
public static JSONObject index(final String volumeId, final String searchPath, final boolean forceIndex) { if (m_interrupt[0]) { LogUtil.log(LogUtil.LogType.INDEX, "Index canceled post interrupt"); m_interrupt[0] = false;// w w w .j a v a 2 s . co m return responseInterruptIndexing(); } OmniFile cacheDir = IndexUtil.getCacheDir(volumeId, searchPath); boolean cacheDirCreated = false; try { cacheDirCreated = OmniUtil.forceMkdir(cacheDir); } catch (IOException e) { return responseFolderCreateError(searchPath); } final String luceneDirPath = cacheDir.getAbsolutePath(); boolean cacheDirExists = !cacheDirCreated; boolean indexingOngoing = m_indexThread != null && m_indexThread.isAlive(); boolean indexingRequired = !cacheDirExists || forceIndex; synchronized (m_lock) { if (indexingOngoing) { if (m_fileTreeActive) m_index_state = INDEX_STATE.filetree; else m_index_state = INDEX_STATE.indexing; } else { if (indexingRequired) m_index_state = INDEX_STATE.indexing; else m_index_state = INDEX_STATE.complete; } } if (indexingRequired || indexingOngoing) { if (indexingOngoing) { // Nothing to do, let the background process run. Monitor m_indexedDocs for progress. } else { synchronized (m_lock) { m_index_state = INDEX_STATE.filetree; m_totalDocs[0] = 0; m_indexedDocs[0] = 0; m_error[0] = ""; } m_threadGroup = new ThreadGroup(INDEX_THREAD_GROUP); m_indexThread = new Thread(m_threadGroup, new Runnable() { @Override public void run() { // Analyzer analyzer = new org.apache.lucene.analysis.core.WhitespaceAnalyzer(); // Analyzer analyzer = new org.apache.lucene.analysis.core.KeywordAnalyzer(); // Analyzer analyzer = new org.apache.lucene.analysis.standard.StandardAnalyzer(); Analyzer analyzer = new org.apache.lucene.analysis.core.SimpleAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); IndexWriter iwriter = null; try { Directory m_directory = FSDirectory.open(Paths.get(luceneDirPath)); iwriter = new IndexWriter(m_directory, config); iwriter.deleteAll(); iwriter.commit(); } catch (IOException e) { LogUtil.logException(LogUtil.LogType.INDEX, e); m_error[0] = "IndexWriter constructor exception"; } synchronized (m_lock) { m_fileTreeActive = true; m_index_state = INDEX_STATE.filetree; } Collection<OmniFile> files = IndexUtil.getFilePaths(volumeId, searchPath); synchronized (m_lock) { m_index_state = INDEX_STATE.indexing; m_fileTreeActive = false; m_totalDocs[0] = files.size(); m_indexedDocs[0] = 0; } try { for (OmniFile file : files) { if (m_interrupt[0]) { LogUtil.log(LogUtil.LogType.INDEX, "Iterator loop canceled"); break; } String path = file.getPath(); // LogUtil.log(LogUtil.LogType.INDEX, "indexing: " + path);// this is a bit excessive iwriter.addDocument(makeDoc(volumeId, path)); synchronized (m_lock) { ++m_indexedDocs[0]; } } iwriter.commit(); iwriter.close(); synchronized (m_lock) { m_index_state = m_interrupt[0] ? INDEX_STATE.interrupted : INDEX_STATE.complete; m_totalDocs[0] = m_indexedDocs[0]; } } catch (Exception e) { LogUtil.logException(LogUtil.LogType.INDEX, e); m_error[0] = "IndexWriter addDocument exception"; } } }, INDEX_THREAD, STACK_SIZE); m_indexThread.setPriority(Thread.MAX_PRIORITY); m_indexThread.start(); } } else { // Indexing is complete // Get number of documents indexed try { Directory directory = FSDirectory.open(Paths.get(luceneDirPath)); DirectoryReader ireader = DirectoryReader.open(directory); synchronized (m_lock) { m_indexedDocs[0] = ireader.numDocs(); m_totalDocs[0] = m_indexedDocs[0]; m_index_state = INDEX_STATE.complete; } ireader.close(); directory.close(); } catch (IOException e) { LogUtil.logException(LogUtil.LogType.INDEX, e); } } JSONObject result = new JSONObject(); try { synchronized (m_lock) { result.put("index_state", m_index_state.toString()); result.put("error", m_error[0]); result.put("indexed_docs", m_indexedDocs[0]); result.put("total_docs", m_totalDocs[0]); // result.put("full_path", cacheDir.getAbsolutePath()); result.put("search_path", searchPath); } } catch (JSONException e) { e.printStackTrace(); } return result; }
From source file:com.orientechnologies.lucene.engine.OLuceneStorage.java
License:Apache License
public void commit() { try {//from www . ja va 2 s. c om OLogManager.instance().info(this, "committing"); final IndexWriter indexWriter = mgrWriter.getIndexWriter(); indexWriter.forceMergeDeletes(); indexWriter.commit(); } catch (IOException e) { OLogManager.instance().error(this, "Error on committing Lucene index", e); } }
From source file:com.orientechnologies.spatial.sandbox.LuceneGeoTest.java
License:Apache License
@Test public void geoIntersectTest() throws IOException, ParseException { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy( new GeohashPrefixTree(JtsSpatialContext.GEO, 11), "location"); strategy.setDistErrPct(0);//from w w w . ja va 2 s. c om IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer()); final RAMDirectory directory = new RAMDirectory(); final IndexWriter writer = new IndexWriter(directory, conf); Shape point = JtsSpatialContext.GEO.getWktShapeParser().parse("POINT (9.4714708 47.6819432)"); Shape polygon = JtsSpatialContext.GEO.getWktShapeParser().parse( "POLYGON((9.481201171875 47.64885294675266,9.471416473388672 47.65128140482982,9.462661743164062 47.64781214443791,9.449443817138672 47.656947367880335,9.445838928222656 47.66110972448931,9.455795288085938 47.667352637215,9.469013214111328 47.67255449415724,9.477081298828125 47.679142768657066,9.490299224853516 47.678680460743834,9.506263732910156 47.679258344995326,9.51364517211914 47.68191653011071,9.518795013427734 47.677177931734406,9.526691436767578 47.679489496903706,9.53390121459961 47.67139857075435,9.50918197631836 47.66180341832901,9.50815200805664 47.6529003141482,9.51192855834961 47.64654002455372,9.504375457763672 47.64237650648966,9.49270248413086 47.649662445325035,9.48617935180664 47.65151268066222,9.481201171875 47.64885294675266))"); Document doc = new Document(); Assert.assertNotEquals(point.relate(polygon), SpatialRelation.INTERSECTS); for (IndexableField f : strategy.createIndexableFields(point)) { doc.add(f); } writer.addDocument(doc); writer.commit(); SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, polygon.getBoundingBox()); Filter filter = strategy.makeFilter(args); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); TopDocs search = searcher.search(new MatchAllDocsQuery(), filter, 1000); Assert.assertEquals(search.totalHits, 0); reader.close(); writer.close(); }
From source file:com.orientechnologies.spatial.sandbox.LuceneGeoTest.java
License:Apache License
@Test public void geoSpeedTest() throws IOException, ParseException { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy( new GeohashPrefixTree(JtsSpatialContext.GEO, 11), "location"); IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer()); final RAMDirectory directory = new RAMDirectory(); final IndexWriter writer = new IndexWriter(directory, conf); Shape multiPolygon = JtsSpatialContext.GEO.getWktShapeParser().parse( "MULTIPOLYGON (((15.520376 38.231155, 15.160243 37.444046, 15.309898 37.134219, 15.099988 36.619987, 14.335229 36.996631, 13.826733 37.104531, 12.431004 37.61295, 12.570944 38.126381, 13.741156 38.034966, 14.761249 38.143874, 15.520376 38.231155)), ((9.210012 41.209991, 9.809975 40.500009, 9.669519 39.177376, 9.214818 39.240473, 8.806936 38.906618, 8.428302 39.171847, 8.388253 40.378311, 8.159998 40.950007, 8.709991 40.899984, 9.210012 41.209991)), ((12.376485 46.767559, 13.806475 46.509306, 13.69811 46.016778, 13.93763 45.591016, 13.141606 45.736692, 12.328581 45.381778, 12.383875 44.885374, 12.261453 44.600482, 12.589237 44.091366, 13.526906 43.587727, 14.029821 42.761008, 15.14257 41.95514, 15.926191 41.961315, 16.169897 41.740295, 15.889346 41.541082, 16.785002 41.179606, 17.519169 40.877143, 18.376687 40.355625, 18.480247 40.168866, 18.293385 39.810774, 17.73838 40.277671, 16.869596 40.442235, 16.448743 39.795401, 17.17149 39.4247, 17.052841 38.902871, 16.635088 38.843572, 16.100961 37.985899, 15.684087 37.908849, 15.687963 38.214593, 15.891981 38.750942, 16.109332 38.964547, 15.718814 39.544072, 15.413613 40.048357, 14.998496 40.172949, 14.703268 40.60455, 14.060672 40.786348, 13.627985 41.188287, 12.888082 41.25309, 12.106683 41.704535, 11.191906 42.355425, 10.511948 42.931463, 10.200029 43.920007, 9.702488 44.036279, 8.888946 44.366336, 8.428561 44.231228, 7.850767 43.767148, 7.435185 43.693845, 7.549596 44.127901, 7.007562 44.254767, 6.749955 45.028518, 7.096652 45.333099, 6.802355 45.70858, 6.843593 45.991147, 7.273851 45.776948, 7.755992 45.82449, 8.31663 46.163642, 8.489952 46.005151, 8.966306 46.036932, 9.182882 46.440215, 9.922837 46.314899, 10.363378 46.483571, 10.442701 46.893546, 11.048556 46.751359, 11.164828 46.941579, 12.153088 47.115393, 12.376485 46.767559)))"); Document doc = new Document(); for (IndexableField f : strategy.createIndexableFields(multiPolygon)) { doc.add(f);//from w w w . j av a 2s. co m } writer.addDocument(doc); writer.commit(); writer.close(); }
From source file:com.orientechnologies.spatial.sandbox.LuceneGeoTest.java
License:Apache License
@Test public void geoSpeedTestInternal() throws IOException, ParseException { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy( new GeohashPrefixTree(JtsSpatialContext.GEO, 11), "location"); IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer()); final RAMDirectory directory = new RAMDirectory(); final IndexWriter writer = new IndexWriter(directory, conf); ODocument entries = loadMultiPolygon(); OMultiPolygonShapeBuilder builder = new OMultiPolygonShapeBuilder(); Shape multiPolygon = builder.fromDoc(entries); Document doc = new Document(); for (IndexableField f : strategy.createIndexableFields(multiPolygon)) { doc.add(f);/*from w w w . jav a 2 s .c o m*/ } writer.addDocument(doc); writer.commit(); writer.close(); }
From source file:com.pjaol.search.test.UnitTests.TestCartesian.java
License:Apache License
private void addData(IndexWriter writer) throws IOException { addPoint(writer, "McCormick & Schmick's Seafood Restaurant", 38.9579000, -77.3572000); addPoint(writer, "Jimmy's Old Town Tavern", 38.9690000, -77.3862000); addPoint(writer, "Ned Devine's", 38.9510000, -77.4107000); addPoint(writer, "Old Brogue Irish Pub", 38.9955000, -77.2884000); addPoint(writer, "Alf Laylah Wa Laylah", 38.8956000, -77.4258000); addPoint(writer, "Sully's Restaurant & Supper", 38.9003000, -77.4467000); addPoint(writer, "TGIFriday", 38.8725000, -77.3829000); addPoint(writer, "Potomac Swing Dance Club", 38.9027000, -77.2639000); addPoint(writer, "White Tiger Restaurant", 38.9027000, -77.2638000); addPoint(writer, "Jammin' Java", 38.9039000, -77.2622000); addPoint(writer, "Potomac Swing Dance Club", 38.9027000, -77.2639000); addPoint(writer, "WiseAcres Comedy Club", 38.9248000, -77.2344000); addPoint(writer, "Glen Echo Spanish Ballroom", 38.9691000, -77.1400000); addPoint(writer, "Whitlow's on Wilson", 38.8889000, -77.0926000); addPoint(writer, "Iota Club and Cafe", 38.8890000, -77.0923000); addPoint(writer, "Hilton Washington Embassy Row", 38.9103000, -77.0451000); addPoint(writer, "HorseFeathers, Bar & Grill", 39.01220000000001, -77.3942); writer.commit(); }
From source file:com.plug.Plug_8_5_2.java
License:Apache License
private void reindexTermbase(DbServer dbServer, HashMap<String, String> companys) throws Exception { log.info("Start upgrading Lucene index for termbase"); TermbaseHandler h = new TermbaseHandler(); List<Termbase> tbs = dbServer.getDbUtil().query(TermbaseHandler.SQL, h); m_analyzer = new NgramAnalyzer(3); for (Termbase tb : tbs) { if (tb.getCOMPANYID().equals(LuceneConstants.SUPER_COMPANY_ID)) { continue; }/*from w w w. j av a 2 s .c o m*/ String cname = companys.get(tb.getCOMPANYID()); File termDir = new File(fileStorageDir, cname + "/TB-" + tb.getTB_NAME()); // check re-indexed if (isIndexedBefore(termDir, tb.getTB_NAME())) { logAlreadyIndex(tb.getTB_NAME()); continue; } showMsg(cname, tb.getTB_NAME(), false); // 1 delete old term base indexes logDeleteFile(termDir.getAbsolutePath()); deleteFile(termDir.getAbsolutePath()); // 2 create new empty dir termDir.mkdirs(); Definition dif = new Definition(tb.getTB_DEFINITION()); List<Index> indexs = dif.getIndexes(); for (Index index : indexs) { // 3 write index into ram RAMDirectory ramdir = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, m_analyzer); config.setOpenMode(OpenMode.CREATE_OR_APPEND); IndexWriter ramIndexWriter = new IndexWriter(ramdir, config); if (index != null && "fuzzy".equalsIgnoreCase(index.getType())) { String folder = index.getLanguageName() + "-" + index.getLocale() + "-TERM"; File indexFolder = new File(termDir, folder); m_directory = indexFolder.getAbsolutePath(); m_fsDir = new SimpleFSDirectory(indexFolder); String sql = TermHandler.generateSQL(tb.getTBID(), index.getLanguageName()); TermHandler termH = new TermHandler(); List<Document> docs = dbServer.getDbUtil().query(sql, termH); for (Document doc : docs) { ramIndexWriter.addDocument(doc); ramIndexWriter.commit(); } // 4 write index from ram into disk IndexWriter diskwriter = getIndexWriter(true); diskwriter.commit(); if (docs != null && docs.size() > 0) { Directory[] ds = new Directory[] { ramdir }; diskwriter.addIndexes(ds); diskwriter.commit(); } // 5 close index writer IOUtils.closeWhileHandlingException(ramIndexWriter); IOUtils.closeWhileHandlingException(diskwriter); ramIndexWriter = null; ramdir = null; } } writeTagFile(termDir, tb.getTB_NAME()); } log.info("End upgrading Lucene index for termbase"); }
From source file:com.plug.Plug_8_5_2.java
License:Apache License
private void queryAndIndexTmdata(DbServer dbServer, HashMap<String, GlobalSightLocale> locales, File tmDir, long tmId, String selectSQL, boolean isTM3, String tm3Id, String tuvTable, String fuzzyTable, int tm3Type) throws SQLException, IOException, Exception { log.info("SQL : " + selectSQL); Connection conn = dbServer.getDbUtil().getConnection(); PreparedStatement preSta = conn.prepareStatement(selectSQL); ResultSet rs = preSta.executeQuery(); IndexWriter diskwriter = null; m_analyzer = null;//from www . j a v a 2 s . c o m GlobalSightLocale lastLocale = null; try { while (rs.next()) { boolean createNew = false; long locale_id = rs.getLong("locale_id"); GlobalSightLocale locale = locales.get(locale_id + ""); if (m_analyzer == null) { createNew = true; } else if (!locale.equals(lastLocale)) { createNew = true; } if (createNew) { m_analyzer = new GsAnalyzer(locale); if (isTM3) { m_analyzer = new GsPerFieldAnalyzer(locale); } lastLocale = locale; if (diskwriter != null) { diskwriter.commit(); IOUtils.closeWhileHandlingException(m_fsDir); IOUtils.closeWhileHandlingException(diskwriter); } File indexFolder = new File(tmDir, locale.toString()); m_directory = indexFolder.getAbsolutePath(); m_fsDir = new SimpleFSDirectory(indexFolder); diskwriter = getIndexWriter(true); log.info("Create new IndexWriter for dir: " + m_directory); } long tuvId = rs.getLong("tuv_id"); String segment = rs.getString("segment_string"); // ignore segment_clob ? if (segment == null || segment.length() == 0) { continue; } segment = LuceneUtil.normalizeTuvData(segment, locale); GlobalSightLocale srcLocale = locales.get(rs.getString("source_locale_id")); String type = rs.getString("type"); String format = rs.getString("format"); long tuId = rs.getLong("tu_id"); boolean isSource = srcLocale.equals(locale); Set<String> targetLocales = null; if (isTM3) { targetLocales = new HashSet<String>(); PreparedStatement psSelectLocales = null; ResultSet rsSelectLocales = null; try { String sql = "select localeId from " + tuvTable + " where tmId=" + tm3Id + " and tuId=" + tuId + " and id<>" + tuvId; psSelectLocales = conn.prepareStatement(sql); rsSelectLocales = psSelectLocales.executeQuery(); while (rsSelectLocales.next()) { long targetlocale_id = rsSelectLocales.getLong("localeId"); GlobalSightLocale targetlocale = locales.get(targetlocale_id + ""); targetLocales.add(targetlocale.toString()); } } finally { dbServer.getDbUtil().closeStatement(psSelectLocales); dbServer.getDbUtil().closeResultSet(rsSelectLocales); } } TuvDocument tuvdoc = new TuvDocument(segment, tuvId, tuId, tmId, isSource, targetLocales, m_analyzer); Document doc = tuvdoc.getDocument(); diskwriter.addDocument(doc); if (isTM3 && fuzzyTable != null) { List<String> tokens = LuceneUtil.createTm3Tokens(segment, locale); List<Long> fps = new ArrayList<Long>(); for (String tok : tokens) { fps.add(Fingerprint.fromString(tok)); } List<Long> fingerprints = new ArrayList<Long>(); fingerprints.add(BOUNDARY); for (Long tok : fps) { fingerprints.add(tok); } fingerprints.add(BOUNDARY); List<Trigram> trigrams = new ArrayList<Trigram>(); for (int i = 0; i + 2 < fingerprints.size(); i++) { trigrams.add( new Trigram(fingerprints.get(i), fingerprints.get(i + 1), fingerprints.get(i + 2))); } List<Long> tset = new ArrayList<Long>(); for (Trigram t : trigrams) { tset.add(t.getValue()); } // add index into database PreparedStatement ps2 = null; try { List<String> keys = new ArrayList<String>(); String sql = null; StringBuilder sb = new StringBuilder("INSERT INTO ").append(fuzzyTable); if (tm3Type == TM3TmType.MULTILINGUAL_SHARED.getId()) { sb.append(" (fingerprint, tuvId, tuId, localeId, tuvCount, isSource) ") .append("VALUES (?, ?, ?, ?, ?, ?)"); sql = sb.toString(); ps2 = conn.prepareStatement(sql, Statement.NO_GENERATED_KEYS); int tuvCount = tset.size(); for (Long fp : tset) { String key = fp + "-" + tuvCount + "-" + locale_id + "-" + (isSource ? "1" : "0") + "-" + tuvId; if (keys.contains(key)) { continue; } ps2.setObject(1, fp); ps2.setObject(2, tuvId); ps2.setObject(3, tuId); ps2.setObject(4, locale_id); ps2.setObject(5, tuvCount); ps2.setObject(6, isSource); ps2.addBatch(); keys.add(key); } } else if (tm3Type == TM3TmType.BILINGUAL.getId()) { sb.append(" (fingerprint, tuvId, tuId, tuvCount, isSource) ") .append("VALUES (?, ?, ?, ?, ?)"); sql = sb.toString(); ps2 = conn.prepareStatement(sql, Statement.NO_GENERATED_KEYS); int tuvCount = tset.size(); for (Long fp : tset) { String key = fp + "-" + tuvCount + "-" + (isSource ? "1" : "0") + "-" + tuvId; if (keys.contains(key)) { continue; } ps2.setObject(1, fp); ps2.setObject(2, tuvId); ps2.setObject(3, tuId); ps2.setObject(4, tuvCount); ps2.setObject(5, isSource); ps2.addBatch(); keys.add(key); } } if (ps2 != null) { ps2.executeBatch(); } } finally { dbServer.getDbUtil().closeStatement(ps2); } } } } finally { if (diskwriter != null) { diskwriter.commit(); } IOUtils.closeWhileHandlingException(m_fsDir); IOUtils.closeWhileHandlingException(diskwriter); dbServer.getDbUtil().closeConn(conn); dbServer.getDbUtil().closeStatement(preSta); dbServer.getDbUtil().closeResultSet(rs); } }
From source file:com.qwazr.search.index.IndexInstance.java
License:Apache License
/** * @param schema/*w w w .j a v a 2 s .com*/ * @param indexDirectory * @return */ final static IndexInstance newInstance(SchemaInstance schema, File indexDirectory, IndexSettingsDefinition settings) throws ServerException, IOException, ReflectiveOperationException, InterruptedException { UpdatableAnalyzer indexAnalyzer = null; UpdatableAnalyzer queryAnalyzer = null; IndexWriter indexWriter = null; Directory dataDirectory = null; try { if (!indexDirectory.exists()) indexDirectory.mkdir(); if (!indexDirectory.isDirectory()) throw new IOException( "This name is not valid. No directory exists for this location: " + indexDirectory); FileSet fileSet = new FileSet(indexDirectory); //Loading the settings if (settings == null) { settings = fileSet.settingsFile.exists() ? JsonMapper.MAPPER.readValue(fileSet.settingsFile, IndexSettingsDefinition.class) : IndexSettingsDefinition.EMPTY; } else { JsonMapper.MAPPER.writeValue(fileSet.settingsFile, settings); } //Loading the fields File fieldMapFile = new File(indexDirectory, FIELDS_FILE); LinkedHashMap<String, FieldDefinition> fieldMap = fieldMapFile.exists() ? JsonMapper.MAPPER.readValue(fieldMapFile, FieldDefinition.MapStringFieldTypeRef) : new LinkedHashMap<>(); //Loading the fields File analyzerMapFile = new File(indexDirectory, ANALYZERS_FILE); LinkedHashMap<String, AnalyzerDefinition> analyzerMap = analyzerMapFile.exists() ? JsonMapper.MAPPER.readValue(analyzerMapFile, AnalyzerDefinition.MapStringAnalyzerTypeRef) : new LinkedHashMap<>(); AnalyzerContext context = new AnalyzerContext(analyzerMap, fieldMap); indexAnalyzer = new UpdatableAnalyzer(context, context.indexAnalyzerMap); queryAnalyzer = new UpdatableAnalyzer(context, context.queryAnalyzerMap); // Open and lock the data directory dataDirectory = FSDirectory.open(fileSet.dataDirectory.toPath()); // Set IndexWriterConfig indexWriterConfig = new IndexWriterConfig(indexAnalyzer); if (settings != null && settings.similarity_class != null) indexWriterConfig.setSimilarity(IndexUtils.findSimilarity(settings.similarity_class)); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); SnapshotDeletionPolicy snapshotDeletionPolicy = new SnapshotDeletionPolicy( indexWriterConfig.getIndexDeletionPolicy()); indexWriterConfig.setIndexDeletionPolicy(snapshotDeletionPolicy); indexWriter = new IndexWriter(dataDirectory, indexWriterConfig); if (indexWriter.hasUncommittedChanges()) indexWriter.commit(); // Finally we build the SearchSearcherManger SearcherManager searcherManager = new SearcherManager(indexWriter, true, null); return new IndexInstance(schema, dataDirectory, settings, analyzerMap, fieldMap, fileSet, indexWriter, searcherManager, queryAnalyzer); } catch (IOException | ServerException | ReflectiveOperationException | InterruptedException e) { // We failed in opening the index. We close everything we can if (queryAnalyzer != null) IOUtils.closeQuietly(queryAnalyzer); if (indexAnalyzer != null) IOUtils.closeQuietly(indexAnalyzer); if (indexWriter != null) IOUtils.closeQuietly(indexWriter); if (dataDirectory != null) IOUtils.closeQuietly(dataDirectory); throw e; } }