List of usage examples for com.mongodb BasicDBObject getDouble
public double getDouble(final String key)
From source file:ch.windmobile.server.mongo.MongoDataSource.java
License:Open Source License
private StationData createStationData(String stationId) throws DataSourceException { BasicDBObject stationJson = findStationJson(stationId); BasicDBObject lastDataJson = (BasicDBObject) stationJson.get("last"); if (lastDataJson == null) { throw new DataSourceException(DataSourceException.Error.INVALID_DATA, "No last data for station '" + stationId + "'"); }//from w w w .j a va 2 s . c o m StationData stationData = new StationData(); stationData.setStationId(stationId); DateTime lastUpdate = getLastUpdateDateTime(lastDataJson); stationData.setLastUpdate(lastUpdate); DateTime now = new DateTime(); DateTime expirationDate = getExpirationDate(now, lastUpdate); stationData.setExpirationDate(expirationDate); // Status stationData.setStatus(getDataStatus(stationJson.getString("status"), now, expirationDate)); // Wind average stationData.setWindAverage((float) lastDataJson.getDouble(DataTypeConstant.windAverage.getJsonKey())); // Wind max stationData.setWindMax((float) lastDataJson.getDouble(DataTypeConstant.windMax.getJsonKey())); List<BasicDBObject> datas = getHistoricData(stationId, lastUpdate, getHistoricDuration()); if (datas.size() > 0) { // Wind direction chart Serie windDirectionSerie = createSerie(datas, DataTypeConstant.windDirection.getJsonKey()); windDirectionSerie.setName(DataTypeConstant.windDirection.getName()); Chart windDirectionChart = new Chart(); windDirectionChart.setDuration(getHistoricDuration()); windDirectionChart.getSeries().add(windDirectionSerie); stationData.setWindDirectionChart(windDirectionChart); // Wind history min/average double minValue = Double.MAX_VALUE; double maxValue = Double.MIN_VALUE; double sum = 0; double[][] windTrendMaxDatas = new double[datas.size()][2]; // double[][] windTrendAverageDatas = new double[windAverageDatas.size()][2]; for (int i = 0; i < datas.size(); i++) { BasicDBObject data = datas.get(i); // JDC unix-time is in seconds, windmobile java-time in millis long millis = data.getLong("_id") * 1000; double windAverage = data.getDouble(DataTypeConstant.windAverage.getJsonKey()); double windMax = data.getDouble(DataTypeConstant.windMax.getJsonKey()); minValue = Math.min(minValue, windAverage); maxValue = Math.max(maxValue, windMax); sum += windAverage; windTrendMaxDatas[i][0] = millis; windTrendMaxDatas[i][1] = windMax; } stationData.setWindHistoryMin((float) minValue); stationData.setWindHistoryAverage((float) (sum / datas.size())); stationData.setWindHistoryMax((float) maxValue); // Wind trend LinearRegression linearRegression = new LinearRegression(windTrendMaxDatas); linearRegression.compute(); double slope = linearRegression.getBeta1(); double angle = Math.toDegrees(Math.atan(slope * getWindTrendScale())); stationData.setWindTrend((int) Math.round(angle)); } // Air temperature stationData.setAirTemperature( (float) lastDataJson.getDouble(DataTypeConstant.airTemperature.getJsonKey(), -1)); // Air humidity stationData.setAirHumidity((float) lastDataJson.getDouble(DataTypeConstant.airHumidity.getJsonKey(), -1)); // Air pressure String key = DataTypeConstant.airPressure.getJsonKey(); if (lastDataJson.containsField(key)) { stationData.setAirPressure((float) lastDataJson.getDouble(key)); } // Rain key = DataTypeConstant.rain.getJsonKey(); if (lastDataJson.containsField(key)) { stationData.setRain((float) lastDataJson.getDouble(key)); } return stationData; }
From source file:com.edgytech.umongo.CollectionPanel.java
License:Apache License
public void shardingDistribution(ButtonBase button) { final DB config = getCollectionNode().getCollection().getDB().getSisterDB("config"); new DbJob() { @Override//from w ww . j a v a2 s . c o m public Object doRun() throws Exception { BasicDBObject result = new BasicDBObject(); BasicDBList shardList = new BasicDBList(); BasicDBObject stats = getStats(); BasicDBObject shards = (BasicDBObject) stats.get("shards"); if (shards == null || shards.isEmpty()) return null; long totalChunks = 0; long totalSize = stats.getLong("size"); long totalCount = stats.getLong("count"); for (Entry shard : shards.entrySet()) { String shardName = (String) shard.getKey(); BasicDBObject shardStats = (BasicDBObject) shard.getValue(); BasicDBObject query = new BasicDBObject("ns", getCollectionNode().getCollection().getFullName()); query.put("shard", shardName); long numChunks = config.getCollection("chunks").count(query); totalChunks += numChunks; double estChunkData = numChunks <= 0 ? 0 : shardStats.getLong("size") / numChunks; long estChunkCount = numChunks <= 0 ? 0 : (long) Math.floor(shardStats.getLong("count") / numChunks); BasicDBObject shardDetails = new BasicDBObject("shard", shardName); shardDetails.put("data", shardStats.getLong("size")); shardDetails.put("pctData", totalSize <= 0 ? 0 : (shardStats.getLong("size") * 100.0) / totalSize); shardDetails.put("docs", shardStats.getLong("count")); shardDetails.put("pctDocs", totalCount <= 0 ? 0 : (shardStats.getLong("count") * 100.0) / totalCount); shardDetails.put("chunks", numChunks); if (shardStats.containsField("avgObjSize")) shardDetails.put("avgDocSize", shardStats.getDouble("avgObjSize")); shardDetails.put("estimatedDataPerChunk", estChunkData); shardDetails.put("estimatedDocsPerChunk", estChunkCount); shardList.add(shardDetails); } result.put("shards", shardList); BasicDBObject total = new BasicDBObject(); total.put("data", totalSize); total.put("docs", totalCount); total.put("chunks", totalChunks); total.put("avgDocSize", stats.getDouble("avgObjSize")); result.put("total", total); return result; } @Override public String getNS() { return getCollectionNode().getCollection().getFullName(); } @Override public String getShortName() { return "Sharding Distribution"; } }.addJob(); }
From source file:com.edgytech.umongo.EditAggGeoNearDialog.java
License:Apache License
@Override public void setParameters(Object value) { BasicDBObject cmd = (BasicDBObject) value; ((DocBuilderField) getBoundUnit(Item.near)).setDBObject((DBObject) cmd.get("near")); setStringFieldValue(Item.distanceField, cmd.getString("distanceField")); setDoubleFieldValue(Item.maxDistance, cmd.getDouble("maxDistance")); if (cmd.containsField("distanceMultiplier")) { setDoubleFieldValue(Item.distanceMultiplier, cmd.getDouble("distanceMultiplier")); }/*from w w w . ja va2s . c o m*/ if (cmd.containsField("query")) { ((DocBuilderField) getBoundUnit(Item.query)).setDBObject((DBObject) cmd.get("query")); } if (cmd.containsField("spherical")) { setBooleanFieldValue(Item.spherical, cmd.getBoolean("spherical")); } if (cmd.containsField("query")) { ((DocBuilderField) getBoundUnit(Item.query)).setDBObject((DBObject) cmd.get("query")); } if (cmd.containsField("includeLocs")) { setStringFieldValue(Item.includeLocs, cmd.getString("includeLocs")); } if (cmd.containsField("uniqueDocs")) { setBooleanFieldValue(Item.uniqueDocs, cmd.getBoolean("uniqueDocs")); } }
From source file:com.edgytech.umongo.ReplicaDialog.java
License:Apache License
public void updateFromReplicaConfig(BasicDBObject member) { // reset/* w w w .j a va2s . co m*/ xmlLoadCheckpoint(); ((TextField) getBoundUnit(Item.host)).editable = false; setStringFieldValue(Item.host, member.getString("host")); setBooleanFieldValue(Item.arbiterOnly, member.getBoolean("arbiterOnly", false)); setBooleanFieldValue(Item.hidden, member.getBoolean("hidden", false)); setBooleanFieldValue(Item.ignoreIndexes, !member.getBoolean("buildIndexes", true)); if (member.containsField("priority")) { setDoubleFieldValue(Item.priority, member.getDouble("priority")); } if (member.containsField("slaveDelay")) { setIntFieldValue(Item.slaveDelay, member.getInt("slaveDelay")); } if (member.containsField("votes")) { setIntFieldValue(Item.votes, member.getInt("votes")); } if (member.containsField("tags")) { ((DocBuilderField) getBoundUnit(Item.tags)).setDBObject((DBObject) member.get("tags")); } }
From source file:com.emuneee.camerasyncmanager.util.DatabaseUtil.java
License:Apache License
/** * Converts BasicDBObject to camera/*from w w w. j a va2 s.co m*/ * @param dbObj * @return */ public Camera dbObjectToCamera(BasicDBObject dbObj) { Camera camera = new Camera(); camera.setId(dbObj.getString("_id")); camera.setTitle(dbObj.getString("title")); camera.setLatitude(dbObj.getDouble("latitude")); camera.setLongitude(dbObj.getDouble("longitude")); camera.setCity(dbObj.getString("city")); camera.setArea(dbObj.getString("area")); camera.setState(dbObj.getString("state")); camera.setZipCode(dbObj.getString("zipcode")); camera.setCountry(dbObj.getString("country")); camera.setUrl(dbObj.getString("url")); camera.setCreated(dbObj.getDate("created")); camera.setUpdated(dbObj.getDate("updated")); return camera; }
From source file:com.ikanow.infinit.e.api.knowledge.processing.ScoringUtils.java
License:Open Source License
@SuppressWarnings("unchecked") private void stage1_initialCountingLoop(DBCursor docs, AdvancedQueryPojo.QueryScorePojo scoreParams, int toReturn, StatisticsPojo scores, LinkedList<BasicDBObject> standaloneEventsReturn, int nCommunities) { double s0_nQuerySubsetDocCountInv = 1.0 / (double) _s0_nQuerySubsetDocCount; // Some memory management: DBCollection dbc = MongoDbManager.getDocument().getMetadata(); DBDecoderFactory defaultDecoder = dbc.getDBDecoderFactory(); try {// w w w . j a va2s . co m SizeReportingBasicBSONDecoder sizeReportingDecoder = new SizeReportingBasicBSONDecoder(); dbc.setDBDecoderFactory(sizeReportingDecoder); long currMemUsage = 0; int ndocs = 0; long lastBatch = 0L; long initialUnusedMemory = Runtime.getRuntime().maxMemory() - Runtime.getRuntime().totalMemory(); long initialFreeMemory = Runtime.getRuntime().freeMemory(); for (DBObject f0 : docs) { BasicDBObject f = (BasicDBObject) f0; long newMemUsage = sizeReportingDecoder.getSize(); if ((newMemUsage - currMemUsage) > 0) { // check every batch long now = new Date().getTime(); //DEBUG //logger.warn(ndocs + " : " + (now - lastBatch) + " : " + newMemUsage + " VS " + Runtime.getRuntime().maxMemory() + " UNUSED " + (Runtime.getRuntime().maxMemory() - Runtime.getRuntime().totalMemory()) + " FREE " + Runtime.getRuntime().freeMemory()); // Check vs total memory: long runtimeMem = Runtime.getRuntime().maxMemory(); // note newMemUsage is the input memory ... gets expanded ~6x by the BSON-ification, allowed at most 1/4rd of memory... // Also if we're taking more than 20s for a batch then limp over the limit and exit... if (((newMemUsage * 24) > runtimeMem) || (((now - lastBatch) > 20000L) && (ndocs >= toReturn))) { long finalUnusedMemory = Runtime.getRuntime().maxMemory() - Runtime.getRuntime().totalMemory(); long finalFreeMemory = Runtime.getRuntime().freeMemory(); logger.error("Query truncated memUsage=" + newMemUsage + ", memory=" + runtimeMem + ", docs=" + ndocs + ", totaldocs=" + scores.found + ", init_free_mem=" + initialFreeMemory + ", end_free_mem=" + finalFreeMemory + ", init_unused_mem=" + initialUnusedMemory + ", end_unused_mem=" + finalUnusedMemory); break; } //TESTED currMemUsage = newMemUsage; lastBatch = now; } //TESTED ndocs++; // Simple handling for standalone events if ((null != _s0_standaloneEventAggregator) && !_s0_bNeedToCalcSig) { //if _s0_bNeedToCalcSig then do this elsewhere ScoringUtils_Associations.addStandaloneEvents(f, 0.0, 0, _s0_standaloneEventAggregator, _s0_bEntityTypeFilterPositive, _s0_bAssocVerbFilterPositive, _s0_entityTypeFilter, _s0_assocVerbFilter, _s0_bEvents, _s0_bSummaries, _s0_bFacts); } //TESTED if (!_s0_bNeedToCalcSig) { continue; } //TESTED if (nCommunities > 1) { // (could have pan-community entities) ObjectId communityId = (ObjectId) f.get(DocumentPojo.communityId_); if (null != communityId) { // (have big problems if so, but anyway!) int retval = _s0_multiCommunityHandler.community_getIdAndInitialize(communityId, _s1_entitiesInDataset); // (returns an int community id but also sets it into the cache, so just use that below) if (Integer.MIN_VALUE == retval) { //this document cannot be viewed from within this set of communities continue; } } } //TESTED TempDocBucket docBucket = new TempDocBucket(); docBucket.dbo = f; ObjectId id = (ObjectId) f.get(DocumentPojo._id_); // If we're going to weight relevance in, or we need the geo temporal decay: if ((0 != scoreParams.relWeight) || (null != scoreParams.timeProx) || (null != scoreParams.geoProx)) { StatisticsPojo.Score scoreObj = scores.getScore().get(id); if (null != scoreObj) { docBucket.explain = scoreObj.explain; // (will normally be null) docBucket.luceneScore = scoreObj.score; if ((null != scoreParams.timeProx) || (null != scoreParams.geoProx)) { if (scoreObj.decay >= 0.0) { docBucket.geoTemporalDecay = scoreObj.decay; } // (see also below for low accuracy geo scoring) } } else { docBucket.luceneScore = 1.0; } } //TESTED else if (this._s0_sortingByDate) { StatisticsPojo.Score scoreObj = scores.getScore().get(id); if (null != scoreObj) { docBucket.nLuceneIndex = scoreObj.nIndex; } } docBucket.manualWeighting = this.getManualScoreWeights(scoreParams, f); BasicDBList l = (BasicDBList) (f.get(DocumentPojo.entities_)); if (null != l) { long nEntsInDoc = l.size(); double dBestGeoScore = 0.0; // (for low accuracy geo only) for (Iterator<?> e0 = l.iterator(); e0.hasNext();) { BasicDBObject e = (BasicDBObject) e0.next(); BasicDBObject tmpGeotag = null; if (_s3_bLowAccuracyGeo || (null != _s1_dManualGeoDecay_latLonInvdecay)) { // low accuracy geo, need to look for geotag tmpGeotag = (BasicDBObject) e.get(EntityPojo.geotag_); } // Get attributes double freq = -1.0; long ntotaldoccount = -1; String entity_index; Double sentiment = null; try { sentiment = (Double) e.get(EntityPojo.sentiment_); ntotaldoccount = e.getLong(EntityPojo.doccount_); freq = e.getDouble(EntityPojo.frequency_); entity_index = e.getString(EntityPojo.index_); if (null == entity_index) { // Just bypass the entity e.put(EntityPojo.significance_, 0.0); nEntsInDoc--; continue; } } catch (Exception ex) { try { String sfreq; if (ntotaldoccount < 0) { sfreq = e.getString(EntityPojo.doccount_); ntotaldoccount = Long.valueOf(sfreq); } if (freq < -0.5) { sfreq = e.getString(EntityPojo.frequency_); freq = Long.valueOf(sfreq).doubleValue(); } entity_index = e.getString(EntityPojo.index_); if (null == entity_index) { // Just bypass the entity e.put(EntityPojo.significance_, 0.0); nEntsInDoc--; continue; } } catch (Exception e2) { // Just bypass the entity e.put(EntityPojo.significance_, 0.0); nEntsInDoc--; continue; } } //TESTED // First loop through is just counting // Retrieve entity (create/initialzie if necessary) EntSigHolder shp = _s1_entitiesInDataset.get(entity_index); if (null == shp) { if (ntotaldoccount > (long) _s0_globalDocCount) { // obviously can't have more entities-in-dos than docs... ntotaldoccount = (long) _s0_globalDocCount; } shp = new EntSigHolder(entity_index, ntotaldoccount, _s0_multiCommunityHandler); // Stage 1a alias handling: set up infrastructure, calculate doc overlap if (null != _s1_aliasLookup) { stage1_initAlias(shp); } if ((null != shp.aliasInfo) && (null == shp.masterAliasSH)) { // this is the discard alias nEntsInDoc--; continue; } //TESTED // Check if entity is in type filter list if (null != _s0_entityTypeFilter) { String entType = null; if (null != shp.aliasInfo) { entType = shp.aliasInfo.getType(); } else { entType = e.getString(EntityPojo.type_); } if (_s0_bEntityTypeFilterPositive) { if ((null != entType) && !_s0_entityTypeFilter.contains(entType.toLowerCase())) { nEntsInDoc--; continue; } } else if ((null != entType) && _s0_entityTypeFilter.contains(entType.toLowerCase())) { //(negative filter) nEntsInDoc--; continue; } } //TESTED (end entity filter) // Geo: if (null != shp.aliasInfo) { if (null != shp.aliasInfo.getGeotag()) { //Geo, overwrite/create tmpGeotag if (_s3_bLowAccuracyGeo || _s3_bExtraAliasGeo || (null != _s1_dManualGeoDecay_latLonInvdecay)) { // Always capture alias geo, even if not in low accuracy mode because we add it to the // legitimate geo: if ((_s3_bLowAccuracyGeo || _s3_bExtraAliasGeo) && (null == _s3_geoBuckets)) { // Initialize the buckets if this is for aggregation not just decay _s3_geoBuckets = (LinkedList<EntSigHolder>[]) new LinkedList[_s3_nGEO_BUCKETS]; } if (null == tmpGeotag) { tmpGeotag = new BasicDBObject(); } tmpGeotag.put(GeoPojo.lat_, shp.aliasInfo.getGeotag().lat); tmpGeotag.put(GeoPojo.lon_, shp.aliasInfo.getGeotag().lon); if (null != shp.aliasInfo.getOntology_type()) { e.put(EntityPojo.ontology_type_, shp.aliasInfo.getOntology_type()); } } } } //TESTED (end geo for aggregation or decay) _s1_entitiesInDataset.put(entity_index, shp); // end Stage 1a alias handling } //(end if is alias) // Stage 1b alias handling: calculate document counts (taking overlaps into account) if (null != shp.masterAliasSH) { // Counts: shp.masterAliasSH.nTotalDocCount++; // docs including overlaps shp.masterAliasSH.avgFreqOverQuerySubset += freq; // Keep track of overlaps: if (f != shp.masterAliasSH.unusedDbo) { shp.masterAliasSH.unusedDbo = f; // (note this is only used in stage 1, alias.unusedDbo is re-used differently in stage 3/4) shp.masterAliasSH.nDocCountInQuerySubset++; // non-overlapping docs ie < shp.nDocCountInQuerySubset } // Sentiment: shp.masterAliasSH.positiveSentiment += shp.positiveSentiment; shp.masterAliasSH.negativeSentiment += shp.negativeSentiment; if (null != sentiment) { shp.masterAliasSH.nTotalSentimentValues++; } } //TESTED (end if is alias) // end Stage 1b // Pan-community logic (this needs to be before the entity object is updated) if (_s0_multiCommunityHandler.isActive()) { _s0_multiCommunityHandler.community_updateCorrelations(shp, ntotaldoccount, entity_index); } else { // (Once we've started multi-community logic, this is no longer desirable) if ((ntotaldoccount > shp.nTotalDocCount) && (ntotaldoccount <= _s0_globalDocCount)) { shp.nTotalDocCount = ntotaldoccount; } //(note there used to be some cases where we adjusted for dc/tf==0, but the // underlying issue in the data model that caused this has been fixed, so it's // now a pathological case that can be ignored) } //(TESTED) // Update counts: _s1_sumFreqInQuerySubset += freq; shp.avgFreqOverQuerySubset += freq; shp.nDocCountInQuerySubset++; shp.decayedDocCountInQuerySubset += docBucket.geoTemporalDecay; // (note this doesn't handle low accuracy geo-decay ... we'll address that via a separate term) TempEntityInDocBucket entBucket = new TempEntityInDocBucket(); entBucket.dbo = e; entBucket.freq = freq; entBucket.doc = docBucket; shp.entityInstances.add(entBucket); if (null != tmpGeotag) { // (only needed for low accuracy geo aggregation) if ((_s3_bLowAccuracyGeo || _s3_bExtraAliasGeo) && (null == shp.geotag)) { // (first time for shp only) shp.geotag = tmpGeotag; shp.geotaggedEntity = e; // (ie for onto type, which has been overwritten in the alias case...) } if (null != _s1_dManualGeoDecay_latLonInvdecay) { // Emulate scripted Lucene calculations double minlat = tmpGeotag.getDouble(GeoPojo.lat_); double minlon = tmpGeotag.getDouble(GeoPojo.lon_); double paramlat = _s1_dManualGeoDecay_latLonInvdecay[0]; double paramlon = _s1_dManualGeoDecay_latLonInvdecay[1]; double gdecay = _s1_dManualGeoDecay_latLonInvdecay[2]; char ontCode = GeoOntologyMapping .encodeOntologyCode(e.getString(EntityPojo.ontology_type_)); double dDecay = QueryDecayScript.getGeoDecay(minlat, minlon, paramlat, paramlon, gdecay, ontCode); if (dDecay > dBestGeoScore) { dBestGeoScore = dDecay; } } //TESTED } //(end if entity has geo and need to process entity geo) if (freq > shp.maxFreq) { shp.maxFreq = freq; } // Sentiment: if ((null != sentiment) && (Math.abs(sentiment) <= 1.1)) { // (actually 1.0) shp.nTotalSentimentValues++; if (sentiment > 0.0) { shp.positiveSentiment += sentiment; } else { shp.negativeSentiment += sentiment; } } else if (null != sentiment) { // corrupt sentiment for some reason?! e.put(EntityPojo.sentiment_, null); } docBucket.docLength += freq; } //(end loop over entities) docBucket.nLeftToProcess = nEntsInDoc; docBucket.nEntsInDoc = (int) nEntsInDoc; if (null != this._s1_dManualGeoDecay_latLonInvdecay) { // Low accuracy geo-calculations docBucket.geoTemporalDecay *= dBestGeoScore; docBucket.luceneScore *= dBestGeoScore; _s2_dAvgLowAccuracyGeoDecay += dBestGeoScore * s0_nQuerySubsetDocCountInv; } //TESTED } // (end if feed has entities) // Handle documents with no entities - can still promote them if (0 == docBucket.nLeftToProcess) { // (use this rather than doc length in case all the entities had freq 0) _s1_noEntityBuckets.add(docBucket); } } // (end loop over feeds) //TESTED } finally { dbc.setDBDecoderFactory(defaultDecoder); } }
From source file:com.ikanow.infinit.e.api.knowledge.processing.ScoringUtils_Associations.java
License:Open Source License
static void finalizeStandaloneEvents(LinkedList<BasicDBObject> standaloneEventList, StandaloneEventHashAggregator standaloneEventAggregator, int nMaxToReturn) { double dMaxSig = (standaloneEventAggregator.dMaxSig + 0.01); // (+0.01 ensures no div by zero and that dBucket<1.0) int nHighPrioAssocs = (standaloneEventAggregator.nPhase0Events + standaloneEventAggregator.nPhase1Events / 4); // (all the docs being promoted and some of the docs that didn't quite make it) int nCurrAssoc = 0; int nFromHighPrioAddedToBucket = 0; for (BasicDBObject assoc : standaloneEventAggregator.tmpList) { try {//www .ja v a2s .co m double dAssocSig = assoc.getDouble(AssociationPojo.assoc_sig_); assoc.put(AssociationPojo.assoc_sig_, Math.sqrt(dAssocSig)); double dBucket = dAssocSig / dMaxSig; int nBucket = StandaloneEventHashAggregator.NUM_BUCKETS_1 - (int) (StandaloneEventHashAggregator.DNUM_BUCKETS * dBucket) % StandaloneEventHashAggregator.NUM_BUCKETS; // (do crazy stuff if dBucket >= 1.0) LinkedList<BasicDBObject> bucketList = standaloneEventAggregator.listBuckets[nBucket]; if (null == bucketList) { bucketList = new LinkedList<BasicDBObject>(); standaloneEventAggregator.listBuckets[nBucket] = bucketList; } bucketList.add(assoc); } catch (Exception e) { // Just ignore that event } if (nCurrAssoc < nHighPrioAssocs) { nFromHighPrioAddedToBucket++; } nCurrAssoc++; // Some exit criteria: if (nFromHighPrioAddedToBucket >= nMaxToReturn) { // Got enough events... if (nCurrAssoc >= nHighPrioAssocs) { // And stepped through the high prio ones break; } } } // (end loop over all collected "events") // Now add the required number of elements to the output list: int nAddedToReturnList = 0; for (LinkedList<BasicDBObject> bucket : standaloneEventAggregator.listBuckets) { if (null != bucket) { for (BasicDBObject dbo : bucket) { if (standaloneEventAggregator.bSimulateAggregation) { dbo = new BasicDBObject(dbo); dbo.remove(AssociationPojo.entity1_); dbo.remove(AssociationPojo.entity2_); dbo.remove(AssociationPojo.verb_); dbo.remove(AssociationPojo.time_start_); dbo.remove(AssociationPojo.time_end_); dbo.remove(AssociationPojo.geo_sig_); } //TESTED standaloneEventList.add(dbo); nAddedToReturnList++; if (nAddedToReturnList >= nMaxToReturn) { return; } } } } }
From source file:com.ikanow.infinit.e.api.knowledge.processing.ScoringUtils_Associations.java
License:Open Source License
static void addStandaloneEvents(BasicDBObject doc, double dDocSig, int nPhase, StandaloneEventHashAggregator standaloneEventAggregator, boolean bEntTypeFilterPositive, boolean bAssocVerbFilterPositive, HashSet<String> entTypeFilter, HashSet<String> assocVerbFilter, boolean bEvents, boolean bSummaries, boolean bFacts) { if (standaloneEventAggregator.bSimulateAggregation) { bSummaries = false;// w w w . j a v a 2 s . com } String sDocIsoPubDate = null; BasicDBList lev = (BasicDBList) (doc.get(DocumentPojo.associations_)); if (null != lev) { for (Iterator<?> e0 = lev.iterator(); e0.hasNext();) { BasicDBObject e = (BasicDBObject) e0.next(); String sEvType = e.getString(AssociationPojo.assoc_type_); boolean bIsFact = false; boolean bIsSummary = false; boolean bKeep = true; if (null == sEvType) { bKeep = false; } else if (sEvType.equalsIgnoreCase("event")) { if (!bEvents) bKeep = false; } else if (sEvType.equalsIgnoreCase("fact")) { if (!bFacts) bKeep = false; bIsFact = true; } else if (sEvType.equalsIgnoreCase("summary")) { if (!bSummaries) bKeep = false; bIsSummary = true; } //TESTED x4 // Filter and aliasing logic: if (bKeep) { boolean bKeep2 = filterAndAliasAssociation(e, standaloneEventAggregator.aliasLookup, true, bEntTypeFilterPositive, bAssocVerbFilterPositive, entTypeFilter, assocVerbFilter); if (!bKeep2) { e0.remove(); // (remove/rename events based on filters where we can, // means we don't have to do it in stage4) bKeep = false; } } //TESTED if (bKeep) { String time_start = null; String time_end = null; // (normally not needed) if (!standaloneEventAggregator.bSimulateAggregation) { //else times are discarded // Add time from document time_start = e.getString(AssociationPojo.time_start_); if (null == time_start) { if (null == sDocIsoPubDate) { // Convert docu pub date to ISO (day granularity): Date pubDate = (Date) doc.get(DocumentPojo.publishedDate_); if (null != pubDate) { SimpleDateFormat f2 = new SimpleDateFormat("yyyy-MM-dd"); time_start = f2.format(pubDate); } } else { time_start = sDocIsoPubDate; // (so it doesn't get added again below) } } //TESTED else { // Remove hourly granularity for consistency time_start = time_start.replaceAll("T.*$", ""); time_end = e.getString(AssociationPojo.time_end_); if (null != time_end) { time_end = time_end.replaceAll("T.*$", ""); } } //TESTED (with debug code, eg time_start = "1997-07-16T19:20:30+01:00") if (null != time_start) { // Ensure it has day granularity, to help with aggregation e.put(AssociationPojo.time_start_, time_start); if (null != time_end) { e.put(AssociationPojo.time_end_, time_end); } } //TESTED } //(end if normal standalone mode, not aggregation simulation) StandaloneEventHashCode evtHolder = new StandaloneEventHashCode( standaloneEventAggregator.bSimulateAggregation, e, bIsSummary, bIsFact); BasicDBObject oldEvt = standaloneEventAggregator.store.get(evtHolder); if (null == oldEvt) { // Doc count (see below) e.put(AssociationPojo.doccount_, 1); double dAssocSig = dDocSig * dDocSig; // Weight down summaries slightly (80%), and summaries with missing entities a lot (50%) if (bIsSummary) { String sEntity2 = (String) e.get(AssociationPojo.entity2_); if (null == sEntity2) { dAssocSig *= 0.50; } else { dAssocSig *= 0.80; } } // Running significance count: e.put(AssociationPojo.assoc_sig_, dAssocSig); // (use sum-squared to score up events that occur frequently) if (dAssocSig > standaloneEventAggregator.dMaxSig) { standaloneEventAggregator.dMaxSig = dAssocSig; } standaloneEventAggregator.store.put(evtHolder, e); // Add to list in some sort of very basic order... if (2 == nPhase) { // Put at the back, it's probably really low sig standaloneEventAggregator.tmpList.add(e); } else if (1 == nPhase) { // Put at the front until Phase 0 comes along standaloneEventAggregator.tmpList.addFirst(e); standaloneEventAggregator.nPhase1Events++; } else { // phases 0 and 1 get the higher orderings standaloneEventAggregator.tmpList.addFirst(e); standaloneEventAggregator.nPhase0Events++; } } else { // Update doc count long nDocCount = oldEvt.getInt(AssociationPojo.doccount_, 1) + 1; oldEvt.put(AssociationPojo.doccount_, nDocCount); // Running significance count: double dAssocSig = oldEvt.getDouble(AssociationPojo.doccount_) + dDocSig * dDocSig; oldEvt.put(AssociationPojo.assoc_sig_, dAssocSig); if (dAssocSig / nDocCount > standaloneEventAggregator.dMaxSig) { standaloneEventAggregator.dMaxSig = dAssocSig; } if (bIsFact && !standaloneEventAggregator.bSimulateAggregation) { // For facts, also update the time range: String old_time_start = oldEvt.getString(AssociationPojo.time_start_); String old_time_end = oldEvt.getString(AssociationPojo.time_end_); // Just keep this really simple and inefficient: TreeSet<String> timeOrder = new TreeSet<String>(); if (null != old_time_start) { timeOrder.add(old_time_start); } if (null != old_time_end) { timeOrder.add(old_time_end); } if (null != time_start) { timeOrder.add(time_start); } if (null != time_end) { timeOrder.add(time_end); } if (timeOrder.size() > 1) { Iterator<String> itStart = timeOrder.iterator(); oldEvt.put(AssociationPojo.time_start_, itStart.next()); Iterator<String> itEnd = timeOrder.descendingIterator(); oldEvt.put(AssociationPojo.time_end_, itEnd.next()); } } // end if is fact - treat times different } //TESTED } // (end if keeping this event) } // (end loop over events) } // (end if this doc has events) }
From source file:com.ikanow.infinit.e.api.knowledge.SearchHandler.java
License:Open Source License
/** * Performs a reverse geolookup, takes a lat/lon and returns a list of nearby * locations/*from ww w . j a va 2 s .c o m*/ * * @param latitude * @param longitude * @return */ private List<SearchSuggestPojo> reverseGeoLookup(Double latitude, Double longitude) { List<SearchSuggestPojo> locations = null; BasicDBList results = runGeoNear(latitude, longitude); if (results != null) { locations = new ArrayList<SearchSuggestPojo>(); if (results.size() > 0) { for (int i = 0; i < 10 && i < results.size(); i++) { BasicDBObject result = (BasicDBObject) results.get(i); Double distance = result.getDouble("dis"); BasicDBObject obj = (BasicDBObject) result.get("obj"); locations.add(buildLocation(obj, distance)); } } } return locations; }
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
/** * Parses the numeric statistics out of given {@link DBObject}. * * @param object the object to parse./*from w w w . j a v a2 s .c om*/ * @return a {@link NumericStatistics} object that wraps the results. */ private NumericStatistics parseNumericStatistics(DBObject object) { NumericStatistics result = null; if (object == null) { result = new NumericStatistics(); } else { BasicDBObject obj = (BasicDBObject) object; long count = obj.getLong("count"); double sum, min, max, avg, std, var; try { sum = obj.getDouble("sum"); } catch (Exception e) { sum = 0; } try { min = obj.getDouble("min"); } catch (Exception e) { min = 0; } try { max = obj.getDouble("max"); } catch (Exception e) { max = 0; } try { avg = obj.getDouble("avg"); } catch (Exception e) { avg = 0; } try { std = obj.getDouble("stddev"); } catch (Exception e) { std = 0; } try { var = obj.getDouble("variance"); } catch (Exception e) { var = 0; } result = new NumericStatistics(count, sum, min, max, avg, std, var); } return result; }