List of usage examples for com.mongodb BulkWriteOperation execute
public BulkWriteResult execute()
From source file:com.camel.realtimelog.PersistenceMongoAccessor.java
License:Open Source License
@Override public void saveAll(List<DBObject> logObjects) throws Exception { BulkWriteOperation operation = logs.initializeUnorderedBulkOperation(); for (DBObject dbObj : logObjects) { operation.insert(dbObj);//from ww w .jav a 2 s. c o m } operation.execute(); //logs.insert(logObjects); logger.debug("save " + logObjects.size() + " errror logs success"); }
From source file:com.edduarte.argus.Context.java
License:Apache License
/** * Indexes the specified document and detects differences between an older * snapshot and the new one. Once differences are collected, saves the resulting * index of all occurrences of the new snapshot for future query and comparison * jobs.//from w w w . j av a 2 s .c o m */ @Override public boolean detectDifferences(String url) { // create a new document snapshot for the provided url DocumentBuilder builder = DocumentBuilder.fromUrl(url).withLanguageDetector(langDetector); if (isStoppingEnabled) { builder.withStopwords(); } if (isStemmingEnabled) { builder.withStemming(); } if (ignoreCase) { builder.ignoreCase(); } Document newDocument = builder.build(occurrencesDB, parserPool); if (newDocument == null) { // A problem occurred during processing, mostly during the fetching phase. // This could happen if the page was unavailable at the time. return false; } // check if there is a older document in the collection Document oldDocument = collection.get(url); if (oldDocument != null) { // there was already a document for this url on the collection, so // detect differences between them and add them to the differences // database DifferenceDetector detector = new DifferenceDetector(oldDocument, newDocument, parserPool); List<Difference> results = detector.call(); removeExistingDifferences(url); DBCollection diffColl = differencesDB.getCollection(url); if (!results.isEmpty()) { BulkWriteOperation bulkOp = diffColl.initializeUnorderedBulkOperation(); results.forEach(bulkOp::insert); bulkOp.execute(); bulkOp = null; diffColl = null; } } // replace the old document in the collection with the new one collection.remove(url); collection.add(newDocument); return true; }
From source file:com.edduarte.argus.document.Document.java
License:Apache License
public void addOccurrences(Iterator<Occurrence> occurrencesIt) { if (!occurrencesIt.hasNext()) { return;/*from www. ja va 2s.c om*/ } BulkWriteOperation builder = occCollection.initializeUnorderedBulkOperation(); while (occurrencesIt.hasNext()) { builder.insert(occurrencesIt.next()); } builder.execute(); builder = null; }
From source file:com.redhat.lightblue.mongo.crud.BasicDocDeleter.java
License:Open Source License
@Override public void delete(CRUDOperationContext ctx, DBCollection collection, DBObject mongoQuery, CRUDDeleteResponse response) {//from w w w . j av a 2 s. c o m LOGGER.debug("Removing docs with {}", mongoQuery); int numDeleted = 0; if (!hookOptimization || ctx.getHookManager().hasHooks(ctx, CRUDOperation.DELETE)) { LOGGER.debug("There are hooks, retrieve-delete"); try (DBCursor cursor = collection.find(mongoQuery, null)) { // Set read preference to primary for read-for-update operations cursor.setReadPreference(ReadPreference.primary()); // All docs, to be put into the context ArrayList<DocCtx> contextDocs = new ArrayList<>(); // ids to delete from the db List<Object> idsToDelete = new ArrayList<>(batchSize); while (cursor.hasNext()) { // We will use this index to access the documents deleted in this batch int thisBatchIndex = contextDocs.size(); if (idsToDelete.size() < batchSize) { // build batch DBObject doc = cursor.next(); DocTranslator.TranslatedDoc tdoc = translator.toJson(doc); DocCtx docCtx = new DocCtx(tdoc.doc, tdoc.rmd); docCtx.setOriginalDocument(docCtx); docCtx.setCRUDOperationPerformed(CRUDOperation.DELETE); contextDocs.add(docCtx); idsToDelete.add(doc.get(MongoCRUDController.ID_STR)); } if (idsToDelete.size() == batchSize || !cursor.hasNext()) { // batch built or run out of documents BulkWriteOperation bw = collection.initializeUnorderedBulkOperation(); for (Object id : idsToDelete) { // doing a bulk of single operations instead of removing by initial query // that way we know which documents were not removed bw.find(new BasicDBObject("_id", id)).remove(); } BulkWriteResult result = null; try { if (writeConcern == null) { LOGGER.debug("Bulk deleting docs"); result = bw.execute(); } else { LOGGER.debug("Bulk deleting docs with writeConcern={} from execution", writeConcern); result = bw.execute(writeConcern); } LOGGER.debug("Bulk deleted docs - attempted {}, deleted {}", idsToDelete.size(), result.getRemovedCount()); } catch (BulkWriteException bwe) { LOGGER.error("Bulk write exception", bwe); handleBulkWriteError(bwe.getWriteErrors(), contextDocs.subList(thisBatchIndex, contextDocs.size())); result = bwe.getWriteResult(); } catch (RuntimeException e) { LOGGER.error("Exception", e); throw e; } finally { numDeleted += result.getRemovedCount(); // clear list before processing next batch idsToDelete.clear(); } } } ctx.setDocumentStream(new ListDocumentStream<DocCtx>(contextDocs)); } } else { LOGGER.debug("There are no hooks, deleting in bulk"); try { if (writeConcern == null) { numDeleted = collection.remove(mongoQuery).getN(); } else { numDeleted = collection.remove(mongoQuery, writeConcern).getN(); } } catch (MongoException e) { LOGGER.error("Deletion error", e); throw e; } ctx.setDocumentStream(new ListDocumentStream<DocCtx>(new ArrayList<DocCtx>())); } response.setNumDeleted(numDeleted); }
From source file:com.redhat.lightblue.mongo.crud.BasicDocSaver.java
License:Open Source License
private void insertDocs(CRUDOperationContext ctx, DBCollection collection, List<DocInfo> list) { if (!list.isEmpty()) { LOGGER.debug("Inserting {} docs", list.size()); if (!md.getAccess().getInsert().hasAccess(ctx.getCallerRoles())) { for (DocInfo doc : list) { doc.inputDoc.addError(/*from ww w . ja va 2 s. co m*/ Error.get("insert", MongoCrudConstants.ERR_NO_ACCESS, "insert:" + md.getName())); } } else { List<DocInfo> insertionAttemptList = new ArrayList<>(list.size()); for (DocInfo doc : list) { Set<Path> paths = roleEval.getInaccessibleFields_Insert(doc.inputDoc); LOGGER.debug("Inaccessible fields:{}", paths); if (paths == null || paths.isEmpty()) { DocTranslator.populateDocHiddenFields(doc.newDoc, md); DocVerUtil.overwriteDocVer(doc.newDoc, docver); insertionAttemptList.add(doc); } else { for (Path path : paths) { doc.inputDoc.addError( Error.get("insert", CrudConstants.ERR_NO_FIELD_INSERT_ACCESS, path.toString())); } } } LOGGER.debug("After access checks, inserting {} docs", insertionAttemptList.size()); if (!insertionAttemptList.isEmpty()) { BulkWriteOperation bw = collection.initializeUnorderedBulkOperation(); for (DocInfo doc : insertionAttemptList) { ctx.getFactory().getInterceptors().callInterceptors(InterceptPoint.PRE_CRUD_INSERT_DOC, ctx, doc.inputDoc); bw.insert(doc.newDoc); doc.inputDoc.setCRUDOperationPerformed(CRUDOperation.INSERT); } try { if (writeConcern == null) { LOGGER.debug("Bulk inserting docs"); bw.execute(); } else { LOGGER.debug("Bulk inserting docs with writeConcern={} from execution", writeConcern); bw.execute(writeConcern); } } catch (BulkWriteException bwe) { LOGGER.error("Bulk write exception", bwe); handleBulkWriteError(bwe.getWriteErrors(), "insert", insertionAttemptList); } catch (RuntimeException e) { LOGGER.error("Exception", e); throw e; } finally { } } } } }
From source file:com.redhat.lightblue.mongo.crud.BatchUpdate.java
License:Open Source License
/** * Runs a batch update using bwo/*from w w w.j a v a2 s .co m*/ * * @param bwo The bulk write operation * @param writeConcern * @param batchSize * @param results The results are populated during this call with an error for each failed doc * @param logger The logger * * @return If returns true, all docs are updated. Otherwise, there * are some failed docs, and concurrent update error detection * should be called */ public static boolean batchUpdate(BulkWriteOperation bwo, WriteConcern writeConcern, int batchSize, Map<Integer, Error> results, Logger logger) { boolean ret = true; BulkWriteResult writeResult; logger.debug("attemptToUpdate={}", batchSize); try { if (writeConcern == null) { writeResult = bwo.execute(); } else { writeResult = bwo.execute(writeConcern); } logger.debug("writeResult={}", writeResult); if (batchSize == writeResult.getMatchedCount()) { logger.debug("Successful update"); } else { logger.warn("notUpdated={}", batchSize - writeResult.getMatchedCount()); ret = false; } } catch (BulkWriteException e) { List<BulkWriteError> writeErrors = e.getWriteErrors(); if (writeErrors != null) { for (BulkWriteError we : writeErrors) { if (MongoCrudConstants.isDuplicate(we.getCode())) { results.put(we.getIndex(), Error.get("update", MongoCrudConstants.ERR_DUPLICATE, we.getMessage())); } else { results.put(we.getIndex(), Error.get("update", MongoCrudConstants.ERR_SAVE_ERROR, we.getMessage())); } } } ret = false; } return ret; }
From source file:com.redhat.lightblue.mongo.crud.MongoSafeUpdateProtocol.java
License:Open Source License
private List<Integer> retryFailedDocs(List<Integer> failedDocs, CommitInfo ci) { List<Integer> newFailedDocs = new ArrayList<>(failedDocs.size()); for (Integer index : failedDocs) { BatchDoc doc = batch.get(index); // Read the doc DBObject findQuery = new BasicDBObject("_id", doc.id); if (cfg.isReevaluateQueryForRetry()) { if (query != null) { List<DBObject> list = new ArrayList<>(2); list.add(findQuery);/*from w ww. j av a2s.c o m*/ list.add(query); findQuery = new BasicDBObject("$and", list); } } DBObject updatedDoc = collection.findOne(findQuery); if (updatedDoc != null) { // if updatedDoc is null, doc is lost. Error remains DBObject newDoc = reapplyChanges(index, updatedDoc); // Make sure reapplyChanges does not insert references // of objects from the old document into the // updatedDoc. That updates both copies of // documents. Use deepCopy if (newDoc != null) { DBObject replaceQuery = writeReplaceQuery(updatedDoc); // Update the doc ver to our doc ver. This doc is here // because its docVer is not set to our docver, so // this is ok DocVerUtil.setDocVer(newDoc, docVer); // Using bulkwrite here with one doc to use the // findAndReplace API, which is lacking in // DBCollection BulkWriteOperation nestedBwo = collection.initializeUnorderedBulkOperation(); nestedBwo.find(replaceQuery).replaceOne(newDoc); try { if (nestedBwo.execute().getMatchedCount() == 1) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Successfully retried to update a doc: replaceQuery={} newDoc={}", replaceQuery, newDoc); } // Successful update ci.errors.remove(index); } } catch (Exception e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Failed retrying to update a doc: replaceQuery={} newDoc={} error={}", replaceQuery, newDoc, e.toString()); } newFailedDocs.add(index); } } else { // reapllyChanges removed the doc from the resultset ci.errors.remove(index); } } else { // Doc no longer exists if (LOGGER.isDebugEnabled()) { LOGGER.debug("Removing doc id={} from retry queue, because it does not exist or match anymore", index); } ci.errors.remove(index); ci.lostDocs.add(index); } } return newFailedDocs; }
From source file:com.staticvillage.recommender.indexer.MongoDBIndexer.java
License:Apache License
@Override public void addBeans(List<Place> beans) throws IndexerException { DBCollection dbCollection = instanceDB.getCollection(collection); BulkWriteOperation builder = dbCollection.initializeUnorderedBulkOperation(); for (Place place : beans) { builder.insert(toDBObject(place)); }//from w ww .j a v a2 s . c o m builder.execute(); }
From source file:com.xoriant.akka.mongodb.bulkimport.actor.MongoInsertionActor.java
License:Apache License
@Override public void onReceive(Object message) throws Exception { if (message instanceof CSVRecordBatchMsg) { CSVRecordBatchMsg csvRecordBatch = (CSVRecordBatchMsg) message; System.out.println("InsertionActor : Batch no " + csvRecordBatch.getBatchNo() + " received ack"); DB db = mongoClient.getDB("akka-bulkimport"); DBCollection personColl = db.getCollection("persons"); BulkWriteOperation builder = personColl.initializeUnorderedBulkOperation(); List<BasicDBObject> persons = csvRecordBatch.getRecords(); for (BasicDBObject personDBObject : persons) { if (validate(personDBObject)) { builder.insert(personDBObject); }/* www . jav a 2 s .co m*/ } BulkWriteResult result = builder.execute(); BatchCompleteMsg batchComplete = new BatchCompleteMsg(csvRecordBatch.getBatchNo(), result.getInsertedCount()); getSender().tell(batchComplete, getSelf()); } else if (message instanceof EndOfFileMsg) { System.out.println("InsertionActor: EOF received"); } else { unhandled(message); } }
From source file:com.zjy.mongo.output.MongoOutputCommitter.java
License:Apache License
@Override public void commitTask(final TaskAttemptContext taskContext) throws IOException { LOG.info("Committing task."); collections = MongoConfigUtil.getOutputCollections(taskContext.getConfiguration()); numberOfHosts = collections.size();//from w w w.j av a 2 s .c o m // Get temporary file. Path tempFilePath = getTaskAttemptPath(taskContext); LOG.info("Committing from temporary file: " + tempFilePath.toString()); long filePos = 0, fileLen; FSDataInputStream inputStream = null; try { FileSystem fs = FileSystem.get(taskContext.getConfiguration()); inputStream = fs.open(tempFilePath); fileLen = fs.getFileStatus(tempFilePath).getLen(); } catch (IOException e) { LOG.error("Could not open temporary file for committing", e); cleanupAfterCommit(inputStream, taskContext); throw e; } int maxDocs = MongoConfigUtil.getBatchSize(taskContext.getConfiguration()); int curBatchSize = 0; DBCollection coll = getDbCollectionByRoundRobin(); BulkWriteOperation bulkOp = coll.initializeOrderedBulkOperation(); // Read Writables out of the temporary file. BSONWritable bw = new BSONWritable(); MongoUpdateWritable muw = new MongoUpdateWritable(); while (filePos < fileLen) { try { // Determine writable type, and perform corresponding operation // on MongoDB. int mwType = inputStream.readInt(); if (MongoWritableTypes.BSON_WRITABLE == mwType) { bw.readFields(inputStream); bulkOp.insert(new BasicDBObject(bw.getDoc().toMap())); } else if (MongoWritableTypes.MONGO_UPDATE_WRITABLE == mwType) { muw.readFields(inputStream); DBObject query = new BasicDBObject(muw.getQuery().toMap()); DBObject modifiers = new BasicDBObject(muw.getModifiers().toMap()); if (muw.isMultiUpdate()) { if (muw.isUpsert()) { bulkOp.find(query).upsert().update(modifiers); } else { bulkOp.find(query).update(modifiers); } } else { if (muw.isUpsert()) { bulkOp.find(query).upsert().updateOne(modifiers); } else { bulkOp.find(query).updateOne(modifiers); } } } else { throw new IOException("Unrecognized type: " + mwType); } filePos = inputStream.getPos(); // Write to MongoDB if the batch is full, or if this is the last // operation to be performed for the Task. if (++curBatchSize >= maxDocs || filePos >= fileLen) { try { bulkOp.execute(); } catch (MongoException e) { LOG.error("Could not write to MongoDB", e); throw e; } coll = getDbCollectionByRoundRobin(); bulkOp = coll.initializeOrderedBulkOperation(); curBatchSize = 0; // Signal progress back to Hadoop framework so that we // don't time out. taskContext.progress(); } } catch (IOException e) { LOG.error("Error reading from temporary file", e); throw e; } } cleanupAfterCommit(inputStream, taskContext); }