List of usage examples for com.mongodb MapReduceCommand setFinalize
public void setFinalize(@Nullable final String finalize)
From source file:com.edgytech.umongo.CollectionPanel.java
License:Apache License
public void mapReduce(final ButtonBase button) { final DBCollection col = getCollectionNode().getCollection(); String map = getStringFieldValue(Item.mrMap); String reduce = getStringFieldValue(Item.mrReduce); String finalize = getStringFieldValue(Item.mrFinalize); String stype = getStringFieldValue(Item.mrType); final OutputType type = OutputType.valueOf(stype.toUpperCase()); String out = getStringFieldValue(Item.mrOut); if (type != OutputType.INLINE && (out.isEmpty())) { new InfoDialog(id, null, null, "Output collection cannot be empty if type is not inline.").show(); return;//from www . j av a2s. co m } String outDB = getStringFieldValue(Item.mrOutDB); DBObject query = ((DocBuilderField) getBoundUnit(Item.mrQuery)).getDBObject(); int limit = getIntFieldValue(Item.mrLimit); final MapReduceCommand cmd = new MapReduceCommand(col, map, reduce, out, type, query); DBObject sort = ((DocBuilderField) getBoundUnit(Item.mrSort)).getDBObject(); if (sort != null) { cmd.setSort(sort); } if (!outDB.isEmpty()) { cmd.setOutputDB(outDB); } if (!finalize.isEmpty()) { cmd.setFinalize(finalize); } if (limit > 0) { cmd.setLimit(limit); } if (getBooleanFieldValue(Item.mrJSMode)) { cmd.addExtraOption("jsMode", true); } final BasicDBObject cmdobj = (BasicDBObject) cmd.toDBObject(); if (getBooleanFieldValue(Item.mrOutSharded)) { ((BasicDBObject) cmdobj.get("out")).put("sharded", true); } if (getBooleanFieldValue(Item.mrNonAtomic)) { ((BasicDBObject) cmdobj.get("out")).put("nonAtomic", true); } new DbJob() { MapReduceOutput output; @Override public Object doRun() { // output = col.mapReduce(cmd); // if type in inline, then query options like slaveOk is fine CommandResult res = null; if (type == MapReduceCommand.OutputType.INLINE) { res = col.getDB().command(cmdobj, col.getOptions()); return res; } res = col.getDB().command(cmdobj); res.throwOnError(); output = new MapReduceOutput(col, cmdobj, res); return output; } @Override public void wrapUp(Object res) { if (output != null) { if (cmd.getOutputType() == OutputType.INLINE) { res = output.results(); } else { // spawn a find doFind(output.getOutputCollection(), null); res = output.getRaw(); } } super.wrapUp(res); } @Override public String getNS() { return col.getFullName(); } @Override public String getShortName() { return "MR"; } @Override public DBObject getRoot(Object result) { return cmdobj; } @Override public ButtonBase getButton() { return button; } @Override DBObject getCommand() { return cmdobj; } @Override DB getDB() { return col.getDB(); } }.addJob(); }
From source file:com.hangum.tadpole.mongodb.core.editors.mapreduce.MapReduceEditor.java
License:Open Source License
/** * execute map reduce/*from w ww .ja va2 s .co m*/ */ private void executeMapReduce() throws Exception { String strMap = textMap.getText(); String strReduce = textReduce.getText(); String strFinilize = textFinalize.getText(); String strOutputTarget = textOutputTarget.getText(); MapReduceCommand.OutputType outputType = (MapReduceCommand.OutputType) comboOutputType .getData(comboOutputType.getText()); DBObject dbQuery = null; if (!"".equals(textQuery.getText())) dbQuery = (DBObject) JSON.parse(textQuery.getText()); DBObject dbSort = null; if (!"".equals(textSort.getText())) dbSort = (DBObject) JSON.parse(textSort.getText()); // . DBCollection dbCol = MongoDBQuery.findCollection(userDB, initColName); MapReduceCommand mrCmd = new MapReduceCommand(dbCol, strMap, strReduce, strOutputTarget, outputType, dbQuery); if (!"".equals(strFinilize)) mrCmd.setFinalize(strFinilize); if (dbSort != null) mrCmd.setSort(dbSort); if (getLimit() > 0) mrCmd.setLimit(getLimit()); if (btnJsMode.getSelection()) mrCmd.addExtraOption("jsMode", true); final BasicDBObject searchObj = (BasicDBObject) mrCmd.toDBObject(); if (btnSharded.getSelection()) ((BasicDBObject) searchObj.get("out")).put("sharded", true); if (btnNoneAtomic.getSelection()) ((BasicDBObject) searchObj.get("out")).put("nonAtomic", true); goMapReduce(dbCol, searchObj, outputType); }
From source file:com.jaspersoft.mongodb.query.MongoDbQueryWrapper.java
License:Open Source License
private void createIterator() throws JRException { if (!queryObject.containsField(COLLECTION_NAME_KEY)) { throw new JRException("\"" + COLLECTION_NAME_KEY + "\" must be part of the query object"); }//from w w w. j a v a 2s. c o m DBObject findQueryObject = (DBObject) queryObject.get(FIND_QUERY_KEY); if (findQueryObject == null) { findQueryObject = new BasicDBObject(); } if (queryObject.containsField(FIND_QUERY_REGEXP_KEY)) { DBObject regExpObject = (DBObject) queryObject.get(FIND_QUERY_REGEXP_KEY); String value, flags; int index; for (String key : regExpObject.keySet()) { value = (String) regExpObject.get(key); if (value.startsWith("/")) { value = value.substring(1, value.length()); } else { throw new JRException("Regular expressions must start with: /"); } if (!value.contains("/")) { throw new JRException("No ending symbol found: /"); } index = value.lastIndexOf("/"); flags = null; if (index == value.length() - 1) { value = value.substring(0, index); } else { flags = value.substring(index + 1, value.length()); value = value.substring(0, index); } findQueryObject.put(key, Pattern.compile((flags != null ? "(?" + flags + ")" : "") + value)); } } DBCollection collection = connection.getMongoDatabase() .getCollectionFromString((String) queryObject.removeField(COLLECTION_NAME_KEY)); if (queryObject.containsField(MAP_REDUCE_KEY)) { Object value = queryObject.removeField(MAP_REDUCE_KEY); if (!(value instanceof DBObject)) { logger.error("MapReduce value must be a valid JSON object"); } else { DBObject mapReduceObject = (DBObject) value; String map = validateProperty(mapReduceObject, MAP_KEY); String reduce = validateProperty(mapReduceObject, REDUCE_KEY); Object outObject = mapReduceObject.get(OUT_KEY); if (outObject == null) { throw new JRException("\"out\" cannot be null"); } String collectionName = null; Object outDb = null; OutputType outputType = null; boolean hasOutputType = false; if (logger.isDebugEnabled()) { logger.debug("Out object: " + outObject + ". Type: " + outObject.getClass().getName()); } if (outObject instanceof String) { collectionName = String.valueOf(outObject); } else if (outObject instanceof DBObject) { DBObject outDbObject = (DBObject) outObject; outDb = outDbObject.removeField(OUT_DB_KEY); Iterator<String> keysIterator = outDbObject.keySet().iterator(); String type = null; if (keysIterator.hasNext()) { type = keysIterator.next(); collectionName = String.valueOf(outDbObject.get(type)); } else { throw new JRException("\"out\" object cannot be empty"); } type = type.toUpperCase(); outputType = OutputType.valueOf(type); if (outputType == null) { throw new JRException("Unknow output type: " + type); } hasOutputType = true; if (logger.isDebugEnabled()) { logger.debug("outobject: " + outDbObject); logger.debug("collectionName: " + collectionName); logger.debug("outputType: " + outputType); } } else { throw new JRException("Unsupported type for \"out\": " + outObject.getClass().getName()); } MapReduceCommand mapReduceCommand = new MapReduceCommand(collection, map, reduce, collectionName, hasOutputType ? outputType : OutputType.REPLACE, null); if (outDb != null) { mapReduceCommand.setOutputDB(String.valueOf(outDb)); } Object finalizeObject = mapReduceObject.removeField(FINALIZE_KEY); if (finalizeObject != null) { mapReduceCommand.setFinalize(String.valueOf(finalizeObject)); } MapReduceOutput mapReduceOutput = collection.mapReduce(mapReduceCommand); DBCollection mapReduceCollection = mapReduceOutput.getOutputCollection(); if (mapReduceCollection != null) { collection = mapReduceCollection; } } } iterator = collection.find(findQueryObject, (DBObject) queryObject.get(FIND_FIELDS_KEY)); if (queryObject.containsField(SORT_KEY)) { iterator = iterator.sort((DBObject) queryObject.get(SORT_KEY)); } if (queryObject.containsField(LIMIT_KEY)) { Integer value = processInteger(queryObject.get(LIMIT_KEY)); if (value != null) { iterator = iterator.limit(value.intValue()); } } }
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduceStats(int key, String property, Filter filter) { LOG.debug("Starting mapReduceStats for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String reduce = ""; String finalize = ""; if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " {\n" + " emit({\n" + " property: property,\n" + " value: property\n" + " }, \n" + " {\n" + " sum: metadataRecord.sourcedValues[0].value,\n" + " min: metadataRecord.sourcedValues[0].value,\n" + " max: metadataRecord.sourcedValues[0].value,\n" + " count: 1,\n" + " diff: 0\n" + " }\n" + " )\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}\n"; reduce = "function reduce(key, values) {\n" + "var a = values[0];\n" + " for (var i = 1; i < values.length; i++) {\n" + " var b = values[i];\n" + " var delta = a.sum / a.count - b.sum / b.count;\n" + " var weight = (a.count * b.count) / (a.count + b.count);\n" + " a.diff += b.diff + delta * delta * weight;\n" + " a.sum = b.sum*1+ a.sum*1;\n" + " a.count += b.count;\n" + " a.min = Math.min(a.min, b.min);\n" + " a.max = Math.max(a.max, b.max);\n" + " }\n" + "return a;" + "}" ;//from ww w . j a v a 2 s . c o m finalize = "function finalize(key, value) {\n" + " value.avg = value.sum / value.count;\n" + " value.variance = value.diff / value.count;\n" + " value.stddev = Math.sqrt(value.variance);\n" + " return value;\n" + "}"; } DBObject query = this.getCachedFilter(filter); LOG.debug("filter query is:\n{}", query); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); cmd.setFinalize(finalize); MapReduceOutput output = elmnts.mapReduce(cmd); //List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("The map-reduce job took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:de.taimos.dvalin.mongo.AbstractMongoDAO.java
License:Apache License
/** * runs a map-reduce-job on the collection. The functions are read from the classpath in the folder mongodb. The systems reads them from * files called <name>.map.js, <name>.reduce.js and optionally <name>.finalize.js. After this the result is converted * using the given {@link MapReduceResultHandler} * * @param <R> the type of the result class * @param name the name of the map-reduce functions * @param query the query to filter the elements used for the map-reduce * @param sort sort query to sort elements before running map-reduce * @param scope the global scope for the JavaScript run * @param conv the converter to convert the result * @return an {@link Iterable} with the result entries * @throws RuntimeException if resources cannot be read *///from ww w . ja v a 2 s. c om protected final <R> Iterable<R> mapReduce(String name, DBObject query, DBObject sort, Map<String, Object> scope, final MapReduceResultHandler<R> conv) { String map = this.getMRFunction(name, "map"); String reduce = this.getMRFunction(name, "reduce"); MapReduceCommand mrc = new MapReduceCommand(this.collection.getDBCollection(), map, reduce, null, OutputType.INLINE, query); String finalizeFunction = this.getMRFunction(name, "finalize"); if (finalizeFunction != null) { mrc.setFinalize(finalizeFunction); } if (sort != null) { mrc.setSort(sort); } if (scope != null) { mrc.setScope(scope); } MapReduceOutput mr = this.collection.getDBCollection().mapReduce(mrc); return new ConverterIterable<R>(mr.results().iterator(), conv); }
From source file:edu.wayne.cs.fms.controller.CRUD.java
public static ArrayList MapReduce(String colName, String map, String reduce, String finalize, MongoClient mongoClient) {//from www . ja v a 2s. c om //MongoClient mongoClient = Connector.connect("localhost", 27017); DB db = mongoClient.getDB("project"); DBCollection temp = db.getCollection(colName); ArrayList result = new ArrayList(); MapReduceCommand cmd = new MapReduceCommand(temp, map, reduce, null, MapReduceCommand.OutputType.INLINE, null); if (finalize != null) { cmd.setFinalize(finalize); } MapReduceOutput out = temp.mapReduce(cmd); for (DBObject o : out.results()) { result.add(o.toString()); } System.out.println("Done"); //mongoClient.close(); return result; }
From source file:org.aw20.mongoworkbench.command.MapReduceMongoCommand.java
License:Open Source License
@SuppressWarnings("deprecation") @Override/*from ww w.j av a2s.c o m*/ public void execute() throws Exception { MongoClient mdb = MongoFactory.getInst().getMongo(sName); if (mdb == null) throw new Exception("no server selected"); if (sDb == null) throw new Exception("no database selected"); MongoFactory.getInst().setActiveDB(sDb); DB db = mdb.getDB(sDb); BasicDBObject cmdMap = parseMongoCommandString(db, cmd); if (!cmdMap.containsField("mapreduceArgs")) throw new Exception("no mapReduce document"); DBCollection collection = db.getCollection(sColl); // Build the Map BasicDBObject options = (BasicDBObject) ((BasicDBList) cmdMap.get("mapreduceArgs")).get(2); String outputCollection = null; String outputDB = null; MapReduceCommand.OutputType outputType = MapReduceCommand.OutputType.INLINE; if (options.get("out") instanceof String) { outputCollection = (String) options.get("out"); outputType = MapReduceCommand.OutputType.REPLACE; } else if (options.get("out") instanceof BasicDBObject) { BasicDBObject out = (BasicDBObject) options.get("out"); if (out.containsField("inline")) { outputCollection = null; } else if (out.containsField("replace")) { outputCollection = (String) out.get("replace"); outputType = MapReduceCommand.OutputType.REPLACE; } else if (out.containsField("merge")) { outputCollection = (String) out.get("merge"); outputType = MapReduceCommand.OutputType.MERGE; } else if (out.containsField("reduce")) { outputCollection = (String) out.get("reduce"); outputType = MapReduceCommand.OutputType.REDUCE; } if (out.containsField("db")) outputDB = (String) out.get("db"); } MapReduceCommand mrc = new MapReduceCommand(collection, ((Code) ((BasicDBList) cmdMap.get("mapreduceArgs")).get(0)).getCode(), ((Code) ((BasicDBList) cmdMap.get("mapreduceArgs")).get(1)).getCode(), outputCollection, outputType, (BasicDBObject) options.get("query")); if (outputDB != null) mrc.setOutputDB(outputDB); if (options.containsField("sort") && options.get("sort") instanceof DBObject) mrc.setSort((DBObject) options.get("sort")); if (options.containsField("scope") && options.get("scope") instanceof DBObject) mrc.setScope(((DBObject) options.get("scope")).toMap()); if (options.containsField("finalize") && options.get("scope") instanceof Code) mrc.setFinalize(((Code) options.get("scope")).getCode()); if (options.containsField("limit")) mrc.setLimit(StringUtil.toInteger(options.get("limit"), -1)); mrc.addExtraOption("jsMode", StringUtil.toBoolean(options.get("jsMode"), false)); mrc.setVerbose(StringUtil.toBoolean(options.get("verbose"), false)); // Run the actual mapreduce function MapReduceOutput mro = collection.mapReduce(mrc); // Pull the inline results if (mro.getOutputCollection() == null) { dbListResult = new BasicDBList(); Iterable<DBObject> it = mro.results(); for (DBObject dbo : it) { dbListResult.add(dbo); } } BasicDBObject dbo = mro.getRaw(); StringBuilder sb = new StringBuilder(); if (dbo.containsField("timeMillis")) sb.append("Time=").append(dbo.get("timeMillis")).append("ms; "); if (dbo.containsField("counts")) { BasicDBObject counts = (BasicDBObject) dbo.get("counts"); sb.append("Counts: input=" + counts.get("input")); sb.append("; emit=" + counts.get("emit")); sb.append("; reduce=" + counts.get("reduce")); sb.append("; output=" + counts.get("output")); } setMessage(sb.toString()); }
From source file:org.basex.modules.MongoDB.java
License:BSD License
/** * Mongodb Mapreduce function with 3 parameters, Map, reduce and query Option. * @param handler Database Handler.//from w ww . j a va2 s .c om * @param col Collection name * @param map Map method * @param reduce Reduce Method * @param query Selection options. * @return Items. * @throws Exception */ public Item mapreduce(final Str handler, final Str col, final Str map, final Str reduce, final Item finalalize, final Item query, final Map options) throws Exception { final DB db = getDbHandler(handler); if (map == null) { throw MongoDBErrors.generalExceptionError("Map function cannot be empty in Mapreduce"); } final DBObject q = query != null ? getDbObjectFromStr(query) : null; final DBCollection collection = db.getCollection(itemToString(col)); String out = null; String outType = null; OutputType op = MapReduceCommand.OutputType.INLINE; if (options != null) { for (Item k : options.keys()) { String key = (String) k.toJava(); if (key.equals("outputs")) { out = (String) options.get(k, null).toJava(); } if (key.equals("outputype")) { outType = (String) options.get(k, null).toJava(); } } if (out != null) { if (outType.toUpperCase().equals("REPLACE")) { op = MapReduceCommand.OutputType.REPLACE; } else if (outType.toUpperCase().equals("MERGE")) { op = MapReduceCommand.OutputType.MERGE; } else if (outType.toUpperCase().equals("REDUCE")) { op = MapReduceCommand.OutputType.REDUCE; } } } db.requestStart(); try { MapReduceCommand cmd = new MapReduceCommand(collection, map.toJava(), reduce.toJava(), out, op, q); if (finalalize != null) { cmd.setFinalize((String) finalalize.toJava()); } final MapReduceOutput outcmd = collection.mapReduce(cmd); return returnResult(handler, Str.get(JSON.serialize(outcmd.results()))); } catch (MongoException e) { throw MongoDBErrors.generalExceptionError(e); } finally { db.requestDone(); } }
From source file:org.basex.modules.MongoDB.java
License:BSD License
/** * Mapreduce all functions in xquery's Map like :{"map":"function(){..}" * , "reduce":"function(){}"}./*from w ww . j ava 2s .c o m*/ * @param handler * @param col collection name * @param options all options of Mapreduce including "map" in key. * @return * @throws Exception */ public Item mapreduce(final Str handler, final Str col, final Map options) throws Exception { if (options == null) { throw MongoDBErrors.generalExceptionError("Map optoins are empty"); } final DB db = getDbHandler(handler); final DBCollection collection = db.getCollection(itemToString(col)); String out = null; String outType = null; String map = null; String reduce = null; DBObject query = null; DBObject sort = null; int limit = 0; String finalalize = null; OutputType op = MapReduceCommand.OutputType.INLINE; for (Item k : options.keys()) { String key = (String) k.toJava(); Value val = options.get(k, null); String value = (String) val.toJava(); if (key.toLowerCase().equals("map")) { map = (String) value; } else if (key.toLowerCase().equals("reduce")) { reduce = value; } else if (key.toLowerCase().equals("outputs")) { out = value; } else if (key.toLowerCase().equals("outputype")) { outType = value; } else if (key.toLowerCase().equals("limit")) { if (val.type().instanceOf(SeqType.ITR_OM)) { long l = ((Item) val).itr(null); limit = (int) l; } else { throw MongoDBErrors.generalExceptionError(" Expected integer Value"); } } else if (key.toLowerCase().equals(SORT)) { sort = getDbObjectFromStr(Str.get(value)); } else if (key.toLowerCase().equals(QUERY)) { query = getDbObjectFromStr(Str.get(value)); } else if (key.toLowerCase().equals(FINALIZE)) { finalalize = value; } } if (out != null && outType != null) { if (outType.toUpperCase().equals("REPLACE")) { op = MapReduceCommand.OutputType.REPLACE; } else if (outType.toUpperCase().equals("MERGE")) { op = MapReduceCommand.OutputType.MERGE; } else if (outType.toUpperCase().equals("REDUCE")) { op = MapReduceCommand.OutputType.REDUCE; } } else if (out != null) { op = MapReduceCommand.OutputType.REPLACE; } if (map == null) { throw MongoDBErrors.generalExceptionError("Map function cannot be empty"); } db.requestStart(); try { MapReduceCommand cmd = new MapReduceCommand(collection, map, reduce, out, op, query); if (finalalize != null) { cmd.setFinalize(finalalize); } if (limit != 0) { cmd.setLimit(limit); } if (sort != null) { cmd.setSort(sort); } final MapReduceOutput outcmd = collection.mapReduce(cmd); return returnResult(handler, Str.get(JSON.serialize(outcmd.results()))); } catch (MongoException e) { throw MongoDBErrors.generalExceptionError(e); } finally { db.requestDone(); } }
From source file:org.basex.modules.nosql.MongoDB.java
License:BSD License
/** * Mongodb Mapreduce function with 5 parameters, Map, reduce and query Option. * @param handler database handler//w w w . ja v a2s . c o m * @param col collection name * @param map Map method * @param reduce Reduce Method * @param finalalize mongodb finalize parameter * @param query Selection options. * @param options additional options * @return Item * @throws Exception exception */ public Item mapreduce(final Str handler, final Str col, final Str map, final Str reduce, final Item finalalize, final Item query, final Map options) throws Exception { final DB db = getDbHandler(handler); if (map == null) { throw MongoDBErrors.generalExceptionError("Map function cannot be empty in Mapreduce"); } final DBObject q = query != null ? getDbObjectFromItem(query) : null; final DBCollection collection = db.getCollection(itemToString(col)); String out = null; String outType = null; OutputType op = MapReduceCommand.OutputType.INLINE; if (options != null) { for (Item k : options.keys()) { String key = (String) k.toJava(); if (key.equals("outputs")) { out = (String) options.get(k, null).toJava(); } if (key.equals("outputype")) { outType = (String) options.get(k, null).toJava(); } } if (out != null) { if (outType.toUpperCase().equals("REPLACE")) { op = MapReduceCommand.OutputType.REPLACE; } else if (outType.toUpperCase().equals("MERGE")) { op = MapReduceCommand.OutputType.MERGE; } else if (outType.toUpperCase().equals("REDUCE")) { op = MapReduceCommand.OutputType.REDUCE; } } } db.requestStart(); try { MapReduceCommand cmd = new MapReduceCommand(collection, map.toJava(), reduce.toJava(), out, op, q); if (finalalize != null) { cmd.setFinalize((String) finalalize.toJava()); } final MapReduceOutput outcmd = collection.mapReduce(cmd); return returnResult(handler, Str.get(JSON.serialize(outcmd.results()))); } catch (MongoException e) { throw MongoDBErrors.generalExceptionError(e); } finally { db.requestDone(); } }