List of usage examples for org.apache.commons.configuration HierarchicalConfiguration getString
public String getString(String key)
From source file:uk.ac.ebi.arrayexpress.components.XmlDbConnectionPool.java
public void initialize() throws Exception { HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance().getPreferences() .getConfSubset("bs.xmldatabase"); if (null != connsConf) { driverXml = connsConf.getString("driver"); // connectionString = connsConf.getString("connectionstring"); connectionString = connsConf.getString("base") + "://" + connsConf.getString("host") + ":" + connsConf.getString("port") + "/" + connsConf.getString("dbname"); logger.debug("connectionString->" + connectionString); } else {/*from w ww . jav a2s .co m*/ logger.error("bs.xmldatabase Configuration is missing!!"); } Class<?> c; try { //TODO: rpe: review this (DB open files) if (coll != null) { coll.close(); db = null; } c = Class.forName(driverXml); // Class<?> c = Class.forName("org.exist.xmldb.DatabaseImpl"); db = (Database) c.newInstance(); DatabaseManager.registerDatabase(db); coll = DatabaseManager.getCollection(connectionString); if (coll == null) { logger.error("Unable to get collection from " + connectionString); } } catch (XMLDBException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } // Receive the database catch (InstantiationException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } catch (IllegalAccessException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } }
From source file:uk.ac.ebi.arrayexpress.utils.saxon.search.AbstractIndexEnvironment.java
public void indexFromXmlDB() throws Exception { // String indexLocationDirectory = ""; String dbHost = ""; String dbPassword = ""; String dbName = ""; int dbPort = 0; // get the default location // indexLocationDirectory = this.indexLocationDirectory; HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance().getPreferences() .getConfSubset("bs.xmldatabase"); if (null != connsConf) { // connectionString = connsConf.getString("connectionstring"); dbHost = connsConf.getString("host"); dbPort = Integer.parseInt(connsConf.getString("port")); dbName = connsConf.getString("dbname"); dbPassword = connsConf.getString("adminpassword"); } else {//from w ww .j a v a2 s . c o m logger.error("bs.xmldatabase Configuration is missing!!"); } indexFromXmlDB(indexLocationDirectory, dbHost, dbPort, dbPassword, dbName); }
From source file:uk.ac.ebi.arrayexpress.utils.saxon.search.AbstractIndexEnvironment.java
public void indexFromXmlDB(String indexLocationDirectory, String dbHost, int dbPort, String dbPassword, String dbName) throws Exception { int countNodes = 0; String driverXml = ""; String connectionString = ""; Collection coll;/*from w w w. ja v a2s .c o m*/ IndexWriter w = null; DirectoryTaxonomyWriter taxoWriter = null; Map<String, XPathExpression> fieldXpe = new HashMap<String, XPathExpression>(); try { Directory indexTempDirectory = FSDirectory.open(new File(indexLocationDirectory, indexId)); logger.debug("Index directory->" + indexLocationDirectory); w = createIndex(indexTempDirectory, indexAnalyzer); Directory taxDir = FSDirectory.open(new File(indexLocationDirectory + "Facets", indexId)); taxoWriter = new DirectoryTaxonomyWriter(taxDir); CategoryDocumentBuilder docBuilder = new CategoryDocumentBuilder(taxoWriter); HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance() .getPreferences().getConfSubset("bs.xmldatabase"); if (null != connsConf) { // TODO: rpe use the component XmlDatabasePooling driverXml = connsConf.getString("driver"); // I will use the connectionString that was passed by parameter // (in several parameters) connectionString = connsConf.getString("base") + "://" + dbHost + ":" + dbPort + "/" + dbName; } else { logger.error("bs.xmldatabase Configuration is missing!!"); } // I cannot register this database again (this is already registered // on XmlDbConnectionPool Component - // java.nio.channels.OverlappingFileLockException // c = Class.forName(driverXml); // db = (Database) c.newInstance(); // DatabaseManager.registerDatabase(db); logger.debug("connectionString->" + connectionString); coll = DatabaseManager.getCollection(connectionString); XPathQueryService service = (XPathQueryService) coll.getService("XPathQueryService", "1.0"); DocumentInfo source = null; // Loop through all result items // collect all the fields data Configuration config = ((SaxonEngine) Application.getAppComponent("SaxonEngine")).trFactory .getConfiguration(); XPath xp = new XPathEvaluator(config); // XPathExpression xpe = xp.compile(this.env.indexDocumentPath); for (FieldInfo field : fields.values()) { fieldXpe.put(field.name, xp.compile(field.path)); logger.debug("Field Path->[{}]", field.path); } // the xmldatabase is not very correct and have memory problem for // queires with huge results, so its necessary to implement our own // iteration mechanism // // // I will collect all the results // ResourceSet set = service.query(this.env.indexDocumentPath); // //TODO rpe // //ResourceSet set = service.query("//Sample"); // logger.debug("Number of results->" + set.getSize()); // long numberResults = set.getSize(); long numberResults = 0; ResourceSet set = service.query("count(" + indexDocumentPath + ")"); if (set.getIterator().hasMoreResources()) { numberResults = Integer.parseInt((String) set.getIterator().nextResource().getContent()); } logger.debug("Number of results->" + numberResults); long pageSizeDefault = 50000; // the samplegroup cannot be big otherwise I will obtain a memory // error ... but the sample must b at least one million because the // paging queries are really slow - we need to balance it // (for samples 1million, for samplegroup 50000) if (numberResults > 1000000) { pageSizeDefault = 1000000; } long pageNumber = 1; int count = 0; // Map<String, AttsInfo[]> cacheAtt = new HashMap<String, // AttsInfo[]>(); // Map<String, XPathExpression> cacheXpathAtt = new HashMap<String, // XPathExpression>(); // Map<String, XPathExpression> cacheXpathAttValue = new // HashMap<String, XPathExpression>(); while ((pageNumber * pageSizeDefault) <= (numberResults + pageSizeDefault - 1)) { // while ((pageNumber<=1)) { // calculate the last hit long pageInit = (pageNumber - 1) * pageSizeDefault + 1; long pageSize = (pageNumber * pageSizeDefault < numberResults) ? pageSizeDefault : (numberResults - pageInit + 1); service = (XPathQueryService) coll.getService("XPathQueryService", "1.0"); // xquery paging using subsequence function long time = System.nanoTime(); // /set = // service.query("for $x in(/Biosamples/SampleGroup/Sample/@id) return string($x)"); // I'm getting everything based on nodeId, because i have the // sample sample in different samplegroups // TODO: change this (just works with baseX) set = service.query("for $x in(subsequence(" + indexDocumentPath + "," + pageInit + "," + pageSize + ")) return db:node-id($x)"); // logger.debug("Number of results of page->" + set.getSize()); double ms = (System.nanoTime() - time) / 1000000d; logger.info("Query XMLDB took ->[{}]", ms); ResourceIterator iter = set.getIterator(); XPath xp2; XPathExpression xpe2; List documentNodes; StringReader reader; // cache of distinct attributes fora each sample group while (iter.hasMoreResources()) { count++; logger.debug("its beeing processed the number ->" + count); String idNode = (String) iter.nextResource().getContent(); //logger.debug("Id node->" + idNode); // I need to get the sample // ResourceSet setid = service.query(indexDocumentPath // + "[@id='" + idSample + "']"); ResourceSet setid = service.query("db:open-id('" + dbName + "'," + idNode + ")"); ResourceIterator iterid = setid.getIterator(); List<CategoryPath> sampleCategories = null; while (iterid.hasMoreResources()) { // System.out.println(""); // /xml=(String) iterid.nextResource().getContent(); // /xml=(String) iter.nextResource().getContent(); // /reader = new StringReader(xml); StringBuilder xml = new StringBuilder(); xml.append((String) iterid.nextResource().getContent()); // logger.debug("xml->"+xml); // logger.debug(xml.toString()); reader = new StringReader(xml.toString()); source = config.buildDocument(new StreamSource(reader)); // logger.debug("XML DB->[{}]", // PrintUtils.printNodeInfo((NodeInfo) source, config)); Document d = new Document(); xp2 = new XPathEvaluator(source.getConfiguration()); int position = indexDocumentPath.lastIndexOf("/"); ; String pathRoot = ""; if (position != -1) { pathRoot = indexDocumentPath.substring(position); } else { pathRoot = indexDocumentPath; } // logger.debug("PathRoot->[{}]",pathRoot); xpe2 = xp2.compile(pathRoot); // TODO rpe // xpe2 = xp2.compile("/Sample"); documentNodes = (List) xpe2.evaluate(source, XPathConstants.NODESET); for (Object node : documentNodes) { try { d = processEntryIndex(node, config, service, fieldXpe); } catch (Exception x) { String xmlError = PrintUtils.printNodeInfo((NodeInfo) node, config); logger.error("XML that was being processed when the error occurred DB->[{}]", xmlError); // to avoid the next running to stop // because its not able to delete the // newSetup directory w.close(); throw new Exception("Xml that is being processed:" + xmlError, x); } } documentNodes = null; source = null; reader = null; xml = null; countNodes++; // logger.debug("count->[{}]", countNodes); // facet tests docBuilder.setCategoryPaths(sampleCategories); docBuilder.build(d); addIndexDocument(w, d); } } logger.debug("until now it were processed->[{}]", pageNumber * pageSizeDefault); pageNumber++; if (coll != null) { try { // coll.close(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } set = null; } setCountDocuments(countNodes); // add metadata to the lucene index Map<String, String> map = new HashMap<String, String>(); map.put("numberDocs", Integer.toString(countNodes)); map.put("date", Long.toString(System.nanoTime())); // logger.debug(Application.getInstance().getComponent("XmlDbConnectionPool").getMetaDataInformation()); // I cannot call directly // getComponent("XmlDbConnectionPool").getMetaDataInformation(), // because I can be working in a did String dbInfo = ((XmlDbConnectionPool) Application.getInstance().getComponent("XmlDbConnectionPool")) .getDBInfo(dbHost, dbPort, dbPassword, dbName); map.put("DBInfo", dbInfo); // facet taxoWriter.commit(); taxoWriter.close(); commitIndex(w, map); } catch (Exception x) { logger.error("Caught an exception:", x); taxoWriter.close(); w.close(); throw x; } }
From source file:uk.ac.ebi.arrayexpress.utils.saxon.search.AbstractIndexEnvironment.java
public void indexIncrementalFromXmlDB() throws Exception { String indexLocationDirectory = ""; String dbHost = ""; String dbPassword = ""; String dbName = ""; int dbPort = 0; // get the default location indexLocationDirectory = indexLocationDirectory; HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance().getPreferences() .getConfSubset("bs.xmldatabase"); if (null != connsConf) { // connectionString = connsConf.getString("connectionstring"); dbHost = connsConf.getString("host"); dbPort = Integer.parseInt(connsConf.getString("port")); dbName = connsConf.getString("dbname"); dbPassword = connsConf.getString("adminpassword"); } else {/*from ww w .j a va 2s . c o m*/ logger.error("bs.xmldatabase Configuration is missing!!"); } indexIncrementalFromXmlDB(indexLocationDirectory, dbHost, dbPort, dbPassword, dbName); }
From source file:uk.ac.ebi.arrayexpress.utils.saxon.search.AbstractIndexEnvironment.java
public void indexIncrementalFromXmlDB(String indexLocationDirectory, String dbHost, int dbPort, String dbPassword, String dbName) throws Exception { // I'm upgrading so the baseline is the current nodes number int countNodes = getCountDocuments(); String driverXml = ""; String connectionString = ""; Collection coll;//from ww w .j a v a2s. c o m IndexWriter w = null; Map<String, XPathExpression> fieldXpe = new HashMap<String, XPathExpression>(); logger.info("indexIncrementalFromXmlDB(generic) is starting for [{}], and initially I have[{}] ... ", new Object[] { indexId, countNodes }); try { Directory indexTempDirectory = FSDirectory.open(new File(indexLocationDirectory, indexId)); w = openIndex(indexTempDirectory, indexAnalyzer); HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance() .getPreferences().getConfSubset("bs.xmldatabase"); if (null != connsConf) { driverXml = connsConf.getString("driver"); connectionString = connsConf.getString("base") + "://" + dbHost + ":" + dbPort + "/" + dbName; } else { logger.error("bs.xmldatabase Configuration is missing!!"); } logger.debug("connectionString->" + connectionString); coll = DatabaseManager.getCollection(connectionString); XPathQueryService service = (XPathQueryService) coll.getService("XPathQueryService", "1.0"); DocumentInfo source = null; Configuration config = ((SaxonEngine) Application.getAppComponent("SaxonEngine")).trFactory .getConfiguration(); XPath xp = new XPathEvaluator(config); for (FieldInfo field : fields.values()) { fieldXpe.put(field.name, xp.compile(field.path)); logger.debug("Field Path->[{}]", field.path); } // the xmldatabase is not very correct and have memory problem for // queires with huge results, so its necessary to implement our own // iteration mechanism // // // I will collect all the results // ResourceSet set = service.query(this.env.indexDocumentPath); long numberResults = 0; ResourceSet set = service.query("count(" + indexDocumentPath + ")"); if (set.getIterator().hasMoreResources()) { numberResults = Integer.parseInt((String) set.getIterator().nextResource().getContent()); } // TODO:######################################Change this after - // this is just a performance test // float percentage=0.1F; // numberResults=Math.round(numberResults * percentage); logger.debug("Number of results->" + numberResults); long pageSizeDefault = 50000; if (numberResults > 1000000) { pageSizeDefault = 1000000; } long pageNumber = 1; int count = 0; Map<String, AttsInfo[]> cacheAtt = new HashMap<String, AttsInfo[]>(); Map<String, XPathExpression> cacheXpathAtt = new HashMap<String, XPathExpression>(); Map<String, XPathExpression> cacheXpathAttValue = new HashMap<String, XPathExpression>(); while ((pageNumber * pageSizeDefault) <= (numberResults + pageSizeDefault - 1)) { // calculate the last hit long pageInit = (pageNumber - 1) * pageSizeDefault + 1; long pageSize = (pageNumber * pageSizeDefault < numberResults) ? pageSizeDefault : (numberResults - pageInit + 1); service = (XPathQueryService) coll.getService("XPathQueryService", "1.0"); // xquery paging using subsequence function long time = System.nanoTime(); // TODO: I'm assuming that there is always an attribute @id in // each element set = service.query("for $x in(subsequence(" + indexDocumentPath + "/@id," + pageInit + "," + pageSize + ")) return string($x)"); double ms = (System.nanoTime() - time) / 1000000d; logger.info("Query XMLDB took ->[{}]", ms); ResourceIterator iter = set.getIterator(); XPath xp2; XPathExpression xpe2; List documentNodes; StringReader reader; // cache of distinct attributes fora each sample group while (iter.hasMoreResources()) { count++; logger.debug("its beeing processed the number ->" + count); String idToProcess = (String) iter.nextResource().getContent(); logger.debug("@id that is being processed->" + idToProcess); // I need to get the sample ResourceSet setid = service.query(indexDocumentPath + "[@id='" + idToProcess + "']"); ResourceIterator iterid = setid.getIterator(); while (iterid.hasMoreResources()) { StringBuilder xml = new StringBuilder(); xml.append((String) iterid.nextResource().getContent()); // logger.debug(xml.toString()); reader = new StringReader(xml.toString()); source = config.buildDocument(new StreamSource(reader)); // logger.debug("XML DB->[{}]", // PrintUtils.printNodeInfo((NodeInfo) source, config)); Document d = new Document(); xp2 = new XPathEvaluator(source.getConfiguration()); int position = indexDocumentPath.lastIndexOf("/"); // TODO: I also need to change this String pathRoot = ""; if (position != -1) { pathRoot = indexDocumentPath.substring(position); } else { pathRoot = indexDocumentPath; } // logger.debug("PathRoot->[{}]",pathRoot); xpe2 = xp2.compile(pathRoot); documentNodes = (List) xpe2.evaluate(source, XPathConstants.NODESET); for (Object node : documentNodes) { // logger.debug("XML DB->[{}]",PrintUtils.printNodeInfo((NodeInfo)node,config)); String idElement = (String) fieldXpe.get("id").evaluate(node, XPathConstants.STRING); // I need to see if it already exists // I will also add this document if it is nor marked // as "todelete" Boolean toDelete = (Boolean) fieldXpe.get("delete").evaluate(node, XPathConstants.BOOLEAN); // TODO:######################################Change // this after - this is just a performance test int deletePercentage = 10; toDelete = (count % deletePercentage) == 0 ? true : false; logger.debug( "Incremental Update - The document [{}] is being processed and is marked to delete?[{}]", new Object[] { idElement, toDelete }); // I will always try to delete the document (i don't // know if it is new or if it was changed) Term idTerm = new Term("id", idElement.toLowerCase()); int countToDelete = getIndexReader().docFreq(idTerm); if (countToDelete > 0) { // if has more than one, I have to send an email // to warn if (countToDelete > 1) { Application.getInstance().sendEmail(null, null, "BIOSAMPLES ERROR - Incremental Update - Removing more than one document! id-> " + idElement, " documents found:" + countToDelete); // I will launch an exception throw new Exception( "BIOSAMPLES ERROR - Incremental Update - Removing more than one document in incremental update id-> " + idElement + " documents found:" + countToDelete); } logger.debug("The document with id [{}] is being deleted from Lucene", idElement); w.deleteDocuments(idTerm); // need to remove one from the number of // documents count countNodes--; } // the element doesn't exist on GUI else { // if it is marked to delete I will just an // warning email - it's possible that it was // inserted and deleted on the Backend but it // had never been sent to the GUI before if (toDelete) { Application.getInstance().sendEmail(null, null, "BIOSAMPLES WARNING - Incremental Update - Id marked for deletion but the id doesn't exist on the GUI! id-> " + idElement, ""); } } // if (toDelete) { // logger.debug( // "The document with id [{}] was marked to deletion so I will not process it", // idElement); // } else { // I just process it is it is not for deletion) if (!toDelete) { try { d = processEntryIndex(node, config, service, fieldXpe); } catch (Exception x) { String xmlError = PrintUtils.printNodeInfo((NodeInfo) node, config); logger.error("XML that was being processed when the error occurred DB->[{}]", xmlError); // to avoid the next running to stop // because its not able to delete the // newSetup directory w.close(); throw new Exception("Xml that is being processed:" + xmlError, x); } countNodes++; addIndexDocument(w, d); } } // } documentNodes = null; source = null; reader = null; xml = null; // logger.debug("count->[{}]", countNodes); } } logger.debug("until now it were processed->[{}]", pageNumber * pageSizeDefault); pageNumber++; // if (coll != null) { // try { // // coll.close(); // } catch (Exception e) { // // TODO Auto-generated catch block // e.printStackTrace(); // } // } set = null; } setCountDocuments(countNodes); // add metadata to the lucene index Map<String, String> map = new HashMap<String, String>(); map.put("numberDocs", Integer.toString(countNodes)); map.put("date", Long.toString(System.nanoTime())); // logger.debug(Application.getInstance().getComponent("XmlDbConnectionPool").getMetaDataInformation()); // I cannot call directly // getComponent("XmlDbConnectionPool").getMetaDataInformation(), // because I can be working in a did String dbInfo = ((XmlDbConnectionPool) Application.getInstance().getComponent("XmlDbConnectionPool")) .getDBInfo(dbHost, dbPort, dbPassword, dbName); // TODO: I need to put here what I have before - to track all the // changes (old numberDocs + old date + oldDBInfo) map.put("DBInfo", dbInfo + "<BR>##################################################<BR>" + getMetadataInformation()); commitIndex(w, map); } catch (Exception x) { logger.error("Caught an exception:", x); w.close(); throw x; } }
From source file:uk.ac.ebi.arrayexpress.utils.saxon.search.AbstractIndexEnvironment.java
public void indexFromXmlDB_FACETS(String indexLocationDirectory, String dbHost, int dbPort, String dbPassword, String dbName) throws Exception { int countNodes = 0; String driverXml = ""; String connectionString = ""; Collection coll;/* w w w . jav a 2 s . c o m*/ IndexWriter w = null; Map<String, XPathExpression> fieldXpe = new HashMap<String, XPathExpression>(); try { Directory indexTempDirectory = FSDirectory.open(new File(indexLocationDirectory, indexId)); w = createIndex(indexTempDirectory, indexAnalyzer); Directory taxDir = FSDirectory.open(new File(indexLocationDirectory + "Facets", indexId)); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxDir); CategoryDocumentBuilder docBuilder = new CategoryDocumentBuilder(taxoWriter); HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance() .getPreferences().getConfSubset("bs.xmldatabase"); if (null != connsConf) { // TODO: rpe use the component XmlDatabasePooling driverXml = connsConf.getString("driver"); // I will use the connectionString that was passed by parameter // (in several parameters) connectionString = connsConf.getString("base") + "://" + dbHost + ":" + dbPort + "/" + dbName; } else { logger.error("bs.xmldatabase Configuration is missing!!"); } // I cannot register this database again (this is already registered // on XmlDbConnectionPool Component - // java.nio.channels.OverlappingFileLockException // c = Class.forName(driverXml); // db = (Database) c.newInstance(); // DatabaseManager.registerDatabase(db); logger.debug("connectionString->" + connectionString); coll = DatabaseManager.getCollection(connectionString); XPathQueryService service = (XPathQueryService) coll.getService("XPathQueryService", "1.0"); DocumentInfo source = null; // Loop through all result items // collect all the fields data Configuration config = ((SaxonEngine) Application.getAppComponent("SaxonEngine")).trFactory .getConfiguration(); XPath xp = new XPathEvaluator(config); // XPathExpression xpe = xp.compile(this.env.indexDocumentPath); for (FieldInfo field : fields.values()) { fieldXpe.put(field.name, xp.compile(field.path)); logger.debug("Field Path->[{}]", field.path); } // the xmldatabase is not very correct and have memory problem for // queires with huge results, so its necessary to implement our own // iteration mechanism // // // I will collect all the results // ResourceSet set = service.query(this.env.indexDocumentPath); // //TODO rpe // //ResourceSet set = service.query("//Sample"); // logger.debug("Number of results->" + set.getSize()); // long numberResults = set.getSize(); long numberResults = 0; ResourceSet set = service.query("count(" + indexDocumentPath + ")"); if (set.getIterator().hasMoreResources()) { numberResults = Integer.parseInt((String) set.getIterator().nextResource().getContent()); } logger.debug("Number of results->" + numberResults); long pageSizeDefault = 50000; // the samplegroup cannot be big otherwise I will obtain a memory // error ... but the sample must b at least one million because the // paging queries are really slow - we need to balance it // (for samples 1million, for samplegroup 50000) if (numberResults > 1000000) { pageSizeDefault = 1000000; } long pageNumber = 1; int count = 0; Map<String, AttsInfo[]> cacheAtt = new HashMap<String, AttsInfo[]>(); Map<String, XPathExpression> cacheXpathAtt = new HashMap<String, XPathExpression>(); Map<String, XPathExpression> cacheXpathAttValue = new HashMap<String, XPathExpression>(); while ((pageNumber * pageSizeDefault) <= (numberResults + pageSizeDefault - 1)) { // while ((pageNumber<=1)) { // calculate the last hit long pageInit = (pageNumber - 1) * pageSizeDefault + 1; long pageSize = (pageNumber * pageSizeDefault < numberResults) ? pageSizeDefault : (numberResults - pageInit + 1); service = (XPathQueryService) coll.getService("XPathQueryService", "1.0"); // xquery paging using subsequence function long time = System.nanoTime(); // /set = // service.query("for $x in(/Biosamples/SampleGroup/Sample/@id) return string($x)"); set = service.query("for $x in(subsequence(" + indexDocumentPath + "/@id," + pageInit + "," + pageSize + ")) return string($x)"); // logger.debug("Number of results of page->" + set.getSize()); double ms = (System.nanoTime() - time) / 1000000d; logger.info("Query XMLDB took ->[{}]", ms); ResourceIterator iter = set.getIterator(); XPath xp2; XPathExpression xpe2; List documentNodes; StringReader reader; // cache of distinct attributes fora each sample group while (iter.hasMoreResources()) { count++; logger.debug("its beeing processed the number ->" + count); String idSample = (String) iter.nextResource().getContent(); logger.debug("idSample->" + idSample); // I need to get the sample ResourceSet setid = service.query(indexDocumentPath + "[@id='" + idSample + "']"); // System.out.println("/Biosamples/SampleGroup/Sample[@id='" // + idSample + "']"); ResourceIterator iterid = setid.getIterator(); List<CategoryPath> sampleCategories = null; while (iterid.hasMoreResources()) { // System.out.println(""); // /xml=(String) iterid.nextResource().getContent(); // /xml=(String) iter.nextResource().getContent(); // logger.debug("xml->"+xml); // /reader = new StringReader(xml); StringBuilder xml = new StringBuilder(); xml.append((String) iterid.nextResource().getContent()); // logger.debug(xml.toString()); reader = new StringReader(xml.toString()); source = config.buildDocument(new StreamSource(reader)); // logger.debug("XML DB->[{}]", // PrintUtils.printNodeInfo((NodeInfo) source, config)); Document d = new Document(); xp2 = new XPathEvaluator(source.getConfiguration()); int position = indexDocumentPath.lastIndexOf("/"); ; String pathRoot = ""; if (position != -1) { pathRoot = indexDocumentPath.substring(position); } else { pathRoot = indexDocumentPath; } // logger.debug("PathRoot->[{}]",pathRoot); xpe2 = xp2.compile(pathRoot); // TODO rpe // xpe2 = xp2.compile("/Sample"); documentNodes = (List) xpe2.evaluate(source, XPathConstants.NODESET); for (Object node : documentNodes) { // logger.debug("XML DB->[{}]",PrintUtils.printNodeInfo((NodeInfo)node,config)); for (FieldInfo field : fields.values()) { try { // Configuration // config=doc.getConfiguration(); // I Just have to calculate the Xpath if (!field.process) { List values = (List) fieldXpe.get(field.name).evaluate(node, XPathConstants.NODESET); // logger.debug("Field->[{}] values-> [{}]", // field.name, // values.toString()); for (Object v : values) { if ("integer".equals(field.type)) { addIntIndexField(d, field.name, v, field.shouldStore, field.shouldSort); // Just to test I will put here // one facet for the samples if (field.name.equalsIgnoreCase("samples")) { System.out.println("Value-->" + v.toString()); sampleCategories = new ArrayList<CategoryPath>(); sampleCategories.add(new CategoryPath("samples", v.toString())); } } else if ("date".equals(field.type)) { // todo: addDateIndexField(d, // field.name, // v); logger.error( "Date fields are not supported yet, field [{}] will not be created", field.name); } else if ("boolean".equals(field.type)) { addBooleanIndexField(d, field.name, v, field.shouldSort); } else { addIndexField(d, field.name, v, field.shouldAnalyze, field.shouldStore, field.shouldSort); } } } else { if (field.name.equalsIgnoreCase("attributes")) { // implement here the biosamples // database sample attributes logic // TODO: rpe // logger.debug("There is A special treatment for this field->" // + field.name); List values = (List) fieldXpe.get(field.name).evaluate(node, XPathConstants.NODESET); // XPathExpression // classAtt=xp.compile("@class"); // XPathExpression // typeAtt=xp.compile("@dataType"); // XPathExpression // valueAtt=xp.compile("value"); String groupId = (String) fieldXpe.get("samplegroup").evaluate(node, XPathConstants.STRING); String id = (String) fieldXpe.get("accession").evaluate(node, XPathConstants.STRING); // logger.debug(groupId+"$$$" + id); // logger.debug("Field->[{}] values-> [{}]", // field.name, // values.toString()); AttsInfo[] attsInfo = null; if (cacheAtt.containsKey(groupId)) { attsInfo = cacheAtt.get(groupId); } else { logger.debug("No exists cache for samplegroup->" + groupId); // ResourceSet setAtt = // service.query("distinct-values(/Biosamples/SampleGroup[@id='" // + groupId + // "']/Sample/attribute[@dataType!='INTEGER']/replace(@class,' ', '-'))"); // /ResourceSet setAtt = // service.query("distinct-values(/Biosamples/SampleGroup[@id='" // + groupId + // "']/Sample/attribute/replace(@class,' ', '-'))"); // /ResourceSet setAtt = // service.query("distinct-values(/Biosamples/SampleGroup[@id='" // + groupId + // "']/Sample/attribute/@class)"); ResourceSet setAtt = service .query("data(/Biosamples/SampleGroup[@id='" + groupId + "']/SampleAttributes/attribute/@class)"); // logger.debug("->" // + // "/Biosamples/SampleGroup[@id='" // + groupId + // "']/SampleAttributes/attribute/@class"); ResourceIterator resAtt = setAtt.getIterator(); int i = 0; attsInfo = new AttsInfo[(int) setAtt.getSize()]; while (resAtt.hasMoreResources()) { String classValue = (String) resAtt.nextResource().getContent(); // logger.debug("->" // + classValue); // need to use this because // of the use of quotes in // the name of the classes String classValueWitoutQuotes = classValue.replaceAll("\"", "\"\""); // logger.debug("Class value->" // + classValue); XPathExpression xpathAtt = null; XPathExpression xpathAttValue = null; if (cacheXpathAtt.containsKey(classValue)) { xpathAtt = cacheXpathAtt.get(classValue); xpathAttValue = cacheXpathAttValue.get(classValue); } else { xpathAtt = xp.compile("./attribute[@class=\"" + classValueWitoutQuotes + "\"]/@dataType"); xpathAttValue = xp.compile( "attribute[@class=\"" + classValueWitoutQuotes + "\"]/value/text()[last()]"); // logger.debug("attribute[@class=\"" // + // classValueWitoutQuotes // + // "\"]//value/text()"); // //xpathAttValue=xp.compile("./attribute[@class=\"" // + // classValueWitoutQuotes // + // "\"]/value[1]/text()"); // logger.debug("./attribute[@class=\"" // + // classValueWitoutQuotes // + // "\"]/value[1]/text()"); cacheXpathAtt.put(classValue, xpathAtt); cacheXpathAttValue.put(classValue, xpathAttValue); } // this doesnt work when the // first sample of sample // group doens have all the // attributes // im using \" becuse there // are some attributes thas // has ' on the name!!! // /ResourceSet setAttType = // service.query("string((/Biosamples/SampleGroup[@id='" // + groupId // +"']/Sample/attribute[@class=replace(\"" // + classValueWitoutQuotes // + // "\",'-',' ')]/@dataType)[1])"); // /ResourceSet setAttType = // service.query("string(/Biosamples/SampleGroup[@id='" // + groupId // +"']/Sample/attribute[@class=\"" // + classValueWitoutQuotes // + "\"]/@dataType)"); ResourceSet setAttType = service .query("data(/Biosamples/SampleGroup[@id='" + groupId + "']/SampleAttributes/attribute[@class=\"" + classValueWitoutQuotes + "\"]/@dataType)"); String dataValue = (String) setAttType.getIterator() .nextResource().getContent(); // logger.debug("Data Type of " // + classValue + " ->" + // dataValue); // String // dataValue=(String)xpathAtt.evaluate(node, // XPathConstants.STRING); AttsInfo attsI = new AttsInfo(classValue, dataValue); // logger.debug("Atttribute->class" // + attsI.name + "->type->" // + attsI.type + "->i" + // i); attsInfo[i] = attsI; // logger.debug("distinct att->" // + value); // cacheAtt.put(groupId, // value); i++; } cacheAtt.put(groupId, attsInfo); // distinctAtt=cacheAtt.get(groupId); // logger.debug("Already exists->" // + distinctAtt); } int len = attsInfo.length; for (int i = 0; i < len; i++) { // logger.debug("$$$$$$->" + // attsInfo[i].name + "$$$$" + // attsInfo[i].type); if (!attsInfo[i].type.equalsIgnoreCase("integer") && !attsInfo[i].type.equalsIgnoreCase("real")) { XPathExpression valPath = cacheXpathAttValue .get(attsInfo[i].name); String val = (String) valPath.evaluate(node, XPathConstants.STRING); // logger.debug("$$$$$$->" + // "STRING->" + val + ""); addIndexField(d, (i + 1) + "", val, true, false, true); } else { XPathExpression valPath = cacheXpathAttValue .get(attsInfo[i].name); String valS = (String) valPath.evaluate(node, XPathConstants.STRING); valS = valS.trim(); // logger.debug("Integer->" // + valS); int val = 0; if (valS == null || valS.equalsIgnoreCase("") || valS.equalsIgnoreCase("NaN")) { valS = "0"; } // sort numbers as strings // logger.debug("class->" + // attsInfo[i].name // +"value->##"+ valS + // "##"); BigDecimal num = new BigDecimal(valS); num = num.multiply(new BigDecimal(100)); int taux = num.toBigInteger().intValue(); valS = String.format("%07d", taux); // logger.debug("Integer->" // + valS + "position->" // +(i+1)+"integer"); addIndexField(d, (i + 1) + "", valS, true, false, true); // addIntIndexField(d, // (i+1)+"integer", new // BigInteger(valS),false, // true); // } } } else { // logger.debug("There is NO special treatment for this field->" // + field.name); } } } catch (XPathExpressionException x) { String xmlError = PrintUtils.printNodeInfo((NodeInfo) node, config); logger.error("XML DB->[{}]", xmlError); logger.error("Caught an exception while indexing expression [" + field.path + "] for document [" + ((NodeInfo) source).getStringValue().substring(0, 20) + "...]", x); throw new Exception("Xml:" + xmlError, x); } } } documentNodes = null; source = null; reader = null; xml = null; countNodes++; // logger.debug("count->[{}]", countNodes); // facet tests docBuilder.setCategoryPaths(sampleCategories); docBuilder.build(d); addIndexDocument(w, d); } } logger.debug("until now it were processed->[{}]", pageNumber * pageSizeDefault); pageNumber++; if (coll != null) { try { // coll.close(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } set = null; } setCountDocuments(countNodes); // add metadata to the lucene index Map<String, String> map = new HashMap<String, String>(); map.put("numberDocs", Integer.toString(countNodes)); map.put("date", Long.toString(System.nanoTime())); // logger.debug(Application.getInstance().getComponent("XmlDbConnectionPool").getMetaDataInformation()); // I cannot call directly // getComponent("XmlDbConnectionPool").getMetaDataInformation(), // because I can be working in a did String dbInfo = ((XmlDbConnectionPool) Application.getInstance().getComponent("XmlDbConnectionPool")) .getDBInfo(dbHost, dbPort, dbPassword, dbName); map.put("DBInfo", dbInfo); // facet taxoWriter.commit(); taxoWriter.close(); commitIndex(w, map); } catch (Exception x) { logger.error("Caught an exception:", x); w.close(); throw x; } }
From source file:uk.ac.ebi.arrayexpress.utils.saxon.search.Configuration.java
private void readConfiguration(HierarchicalConfiguration config) { List indexList = config.configurationsAt("index"); for (Object conf : indexList) { HierarchicalConfiguration indexConfig = (HierarchicalConfiguration) conf; String indexId = indexConfig.getString("[@id]"); this.indicesConfig.put(indexId, indexConfig); }/*from w ww.j a va2 s. c om*/ }
From source file:uk.ac.ebi.atlas.model.baseline.BaselineExperimentConfiguration.java
public Set<Factor> getDefaultFilterFactors() { Set<Factor> defaultFilterFactors = new HashSet<>(); List<HierarchicalConfiguration> fields = config.configurationsAt("defaultFilterFactors.filterFactor"); for (HierarchicalConfiguration sub : fields) { String factorType = sub.getString("type"); String factorValue = sub.getString("value"); defaultFilterFactors.add(new Factor(factorType, factorValue)); }// w w w . j av a 2 s. c om return defaultFilterFactors; }
From source file:uk.ac.ebi.atlas.model.baseline.BaselineExperimentConfiguration.java
public Map<String, String> getSpeciesMapping() { Map<String, String> mapping = new HashMap<>(); List<HierarchicalConfiguration> fields = config.configurationsAt("speciesMapping.mapping"); for (HierarchicalConfiguration sub : fields) { String samples = sub.getString("samples").toLowerCase(); String genes = sub.getString("genes").toLowerCase(); mapping.put(samples, genes);/*from w w w. j ava 2 s . co m*/ } return mapping; }
From source file:uk.ac.ebi.fg.biostudies.components.XmlDbConnectionPool.java
public void initialize() throws Exception { HierarchicalConfiguration connsConf = (HierarchicalConfiguration) Application.getInstance().getPreferences() .getConfSubset("bs.xmldatabase"); if (null != connsConf) { driverXml = connsConf.getString("driver"); // connectionString = connsConf.getString("connectionstring"); connectionString = connsConf.getString("base") + "://" + connsConf.getString("host") + ":" + connsConf.getString("port") + "/" + connsConf.getString("dbname"); logger.debug("connectionString->" + connectionString); } else {// w w w . j a v a2 s. co m logger.error("bs.xmldatabase Configuration is missing!!"); } Class<?> c; try { //TODO: rpe: review this (DB open files) if (coll != null) { coll.close(); db = null; } c = Class.forName(driverXml); // Class<?> c = Class.forName("org.exist.xmldb.DatabaseImpl"); db = (Database) c.newInstance(); DatabaseManager.registerDatabase(db); coll = DatabaseManager.getCollection(connectionString); } catch (XMLDBException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } // Receive the database catch (InstantiationException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } catch (IllegalAccessException e) { // TODO Auto-generated catch block logger.error("Exception:->[{}]", e.getMessage()); e.printStackTrace(); } }