List of usage examples for java.util.stream Stream sorted
Stream<T> sorted();
From source file:com.joyent.manta.client.multipart.ServerSideMultipartManager.java
/** * Creates the JSON request body used to commit all of the parts of a multipart * upload request.// w w w. j a va 2 s .c o m * * @param parts stream of tuples - this is a terminal operation that will close the stream * @return byte array containing JSON data */ static ImmutablePair<byte[], Integer> createCommitRequestBody( final Stream<? extends MantaMultipartUploadTuple> parts) { final JsonNodeFactory nodeFactory = MantaObjectMapper.NODE_FACTORY_INSTANCE; final ObjectNode objectNode = new ObjectNode(nodeFactory); final ArrayNode partsArrayNode = new ArrayNode(nodeFactory); objectNode.set("parts", partsArrayNode); try (Stream<? extends MantaMultipartUploadTuple> sorted = parts.sorted()) { sorted.forEach(tuple -> partsArrayNode.add(tuple.getEtag())); } Validate.isTrue(partsArrayNode.size() > 0, "Can't commit multipart upload with no parts"); try { return ImmutablePair.of(MantaObjectMapper.INSTANCE.writeValueAsBytes(objectNode), partsArrayNode.size()); } catch (IOException e) { String msg = "Error serializing JSON for commit MPU request body"; throw new MantaMultipartException(msg, e); } }
From source file:com.joyent.manta.client.MantaClientFindIT.java
/** * This test determines that we are filtering results as per our expection * when using a filter predicate with find(). *//*from ww w . j a v a 2 s . co m*/ public void canFindRecursivelyWithFilter() throws IOException { List<String> level1Dirs = Arrays.asList(testPathPrefix + "aaa_bbb_ccc", testPathPrefix + "aaa_111_ccc", testPathPrefix + UUID.randomUUID()); List<String> level1Files = Arrays.asList(testPathPrefix + UUID.randomUUID(), testPathPrefix + "aaa_222_ccc"); for (String dir : level1Dirs) { mantaClient.putDirectory(dir); } for (String file : level1Files) { mantaClient.put(file, TEST_DATA, StandardCharsets.UTF_8); } List<String> level2Files = level1Dirs.stream().flatMap(dir -> Stream.of(dir + SEPARATOR + "aaa_333_ccc", dir + SEPARATOR + "aaa_444_ccc", dir + SEPARATOR + UUID.randomUUID())).collect(Collectors.toList()); for (String file : level2Files) { mantaClient.put(file, TEST_DATA, StandardCharsets.UTF_8); } final String[] results; Predicate<? super MantaObject> filter = (Predicate<MantaObject>) obj -> FilenameUtils.getName(obj.getPath()) .startsWith("aaa_"); try (Stream<MantaObject> stream = mantaClient.find(testPathPrefix, filter)) { Stream<String> paths = stream.map(MantaObject::getPath); Stream<String> sorted = paths.sorted(); results = sorted.toArray(String[]::new); } String[] expected = new String[] { testPathPrefix + "aaa_111_ccc", testPathPrefix + "aaa_111_ccc" + SEPARATOR + "aaa_333_ccc", testPathPrefix + "aaa_111_ccc" + SEPARATOR + "aaa_444_ccc", testPathPrefix + "aaa_222_ccc", testPathPrefix + "aaa_bbb_ccc", testPathPrefix + "aaa_bbb_ccc" + SEPARATOR + "aaa_333_ccc", testPathPrefix + "aaa_bbb_ccc" + SEPARATOR + "aaa_444_ccc", }; try { Assert.assertEqualsNoOrder(results, expected); } catch (AssertionError e) { System.err.println("ACTUAL: " + StringUtils.join(results, ", ")); System.err.println("EXPECTED: " + StringUtils.join(expected, ", ")); throw e; } }
From source file:com.joyent.manta.client.multipart.TestMultipartManager.java
@Override public void complete(TestMultipartUpload upload, Stream<? extends MantaMultipartUploadTuple> partsStream) throws IOException { Validate.notNull(upload, "Upload state object must not be null"); File objectContents = upload.getContents(); try (Stream<? extends MantaMultipartUploadTuple> sorted = partsStream.sorted(); FileOutputStream fout = new FileOutputStream(objectContents)) { sorted.forEach(tuple -> {/*from w ww. jav a 2s . c o m*/ final int partNumber = tuple.getPartNumber(); File part = new File(upload.getPartsPath() + File.separator + partNumber); try (FileInputStream fin = new FileInputStream(part)) { IOUtils.copy(fin, fout); } catch (IOException e) { throw new UncheckedIOException(e); } }); } if (upload.getContentLength() != null) { Validate.isTrue(upload.getContentLength().equals(objectContents.length()), "Completed object's content length [%d] didn't equal expected length [%d]", objectContents.length(), upload.getContentLength()); } }
From source file:org.apache.taverna.databundle.TestDataBundles.java
@Test public void resolveStream() throws Exception { Path inputs = DataBundles.getInputs(dataBundle); Path list = DataBundles.getPort(inputs, "in1"); DataBundles.createList(list);//from w ww . j a v a 2 s .c o m Path nested0 = DataBundles.newListItem(list); DataBundles.newListItem(nested0); DataBundles.setStringValue(DataBundles.newListItem(nested0), "test0,0"); DataBundles.setStringValue(DataBundles.newListItem(nested0), "test0,1"); DataBundles.setStringValue(DataBundles.newListItem(nested0), "test0,2"); DataBundles.setError(DataBundles.newListItem(nested0), "Ignore me", "This error is hidden"); Path nested1 = DataBundles.newListItem(list); DataBundles.newListItem(nested1); // empty Path nested2 = DataBundles.newListItem(list); DataBundles.newListItem(nested2); DataBundles.setStringValue(DataBundles.newListItem(nested2), "test2,0"); DataBundles.setReference(DataBundles.newListItem(nested2), URI.create("http://example.com/")); assertEquals(6, DataBundles.resolveAsStream(list, Object.class).count()); assertEquals(6, DataBundles.resolveAsStream(list, Path.class).count()); assertEquals(5, DataBundles.resolveAsStream(list, URI.class).count()); assertEquals(1, DataBundles.resolveAsStream(list, URL.class).count()); assertEquals(0, DataBundles.resolveAsStream(list, File.class).count()); assertEquals(1, DataBundles.resolveAsStream(list, ErrorDocument.class).count()); // Let's have a look at one of the types in detail assertEquals(4, DataBundles.resolveAsStream(list, String.class).count()); Stream<String> resolved = DataBundles.resolveAsStream(list, String.class); Object[] strings = resolved.sorted().map(t -> t.replace("test", "X")).toArray(); // NOTE: We can only assume the below order because we used .sorted() assertEquals("X0,0", strings[0]); assertEquals("X0,1", strings[1]); assertEquals("X0,2", strings[2]); assertEquals("X2,0", strings[3]); }
From source file:com.joyent.manta.client.multipart.JobsMultipartManager.java
/** * Completes a multipart transfer by assembling the parts on Manta. * This is an asynchronous operation and you will need to call * {@link #waitForCompletion(MantaMultipartUpload, Duration, int, Function)} * to block until the operation completes. * * @param upload multipart upload object * @param partsStream stream of multipart part objects * @throws IOException thrown if there is a problem connecting to Manta *///from w ww.j av a 2 s . c o m @Override public void complete(final JobsMultipartUpload upload, final Stream<? extends MantaMultipartUploadTuple> partsStream) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); LOGGER.debug("Completing multipart upload [{}]", upload.getId()); final String uploadDir = multipartUploadDir(upload.getId()); final MultipartMetadata metadata = downloadMultipartMetadata(upload.getId()); final Map<String, MantaMultipartUploadPart> listing = new HashMap<>(); try (Stream<MantaMultipartUploadPart> listStream = listParts(upload).limit(getMaxParts())) { listStream.forEach(p -> listing.put(p.getEtag(), p)); } final String path = metadata.getPath(); final StringBuilder jobExecText = new StringBuilder("set -o pipefail; mget -q "); List<MantaMultipartUploadTuple> missingTuples = new ArrayList<>(); final AtomicInteger count = new AtomicInteger(0); try (Stream<? extends MantaMultipartUploadTuple> distinct = partsStream.sorted().distinct()) { distinct.forEach(part -> { final int i = count.incrementAndGet(); if (i > getMaxParts()) { String msg = String.format( "Too many multipart parts specified [%d]. " + "The maximum number of parts is %d", getMaxParts(), count.get()); throw new IllegalArgumentException(msg); } // Catch and log any gaps in part numbers if (i != part.getPartNumber()) { missingTuples.add(new MantaMultipartUploadTuple(i, "N/A")); } else { final MantaMultipartUploadPart o = listing.get(part.getEtag()); if (o != null) { jobExecText.append(o.getObjectPath()).append(" "); } else { missingTuples.add(part); } } }); } if (!missingTuples.isEmpty()) { final MantaMultipartException e = new MantaMultipartException( "Multipart part(s) specified couldn't be found"); int missingCount = 0; for (MantaMultipartUploadTuple missingPart : missingTuples) { String key = String.format("missing_part_%d", ++missingCount); e.setContextValue(key, missingPart.toString()); } throw e; } final String headerFormat = "\"%s: %s\" "; jobExecText.append("| mput ").append("-H ") .append(String.format(headerFormat, UPLOAD_ID_METADATA_KEY, upload.getId())).append("-H ") .append(String.format(headerFormat, JOB_ID_METADATA_KEY, "$MANTA_JOB_ID")).append("-q "); if (metadata.getContentType() != null) { jobExecText.append("-H 'Content-Type: ").append(metadata.getContentType()).append("' "); } MantaMetadata objectMetadata = metadata.getObjectMetadata(); if (objectMetadata != null) { Set<Map.Entry<String, String>> entries = objectMetadata.entrySet(); for (Map.Entry<String, String> entry : entries) { jobExecText.append("-H '").append(entry.getKey()).append(": ").append(entry.getValue()) .append("' "); } } jobExecText.append(path); final MantaJobPhase concatPhase = new MantaJobPhase().setType("reduce").setExec(jobExecText.toString()); final MantaJobPhase cleanupPhase = new MantaJobPhase().setType("reduce").setExec("mrm -r " + uploadDir); MantaJobBuilder.Run run = mantaClient.jobBuilder().newJob(String.format(JOB_NAME_FORMAT, upload.getId())) .addPhase(concatPhase).addPhase(cleanupPhase).run(); // We write the job id to Metadata object so that we can query it easily writeJobIdToMetadata(upload.getId(), run.getId()); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Created job for concatenating parts: {}", run.getId()); } }
From source file:com.ikanow.aleph2.analytics.services.TestDeduplicationService.java
@SuppressWarnings("unchecked") @Test/*from w w w. ja va 2s. c o m*/ public void test_handleDuplicateRecord() { final IEnrichmentModuleContext enrich_context = Mockito.mock(IEnrichmentModuleContext.class); Mockito.when(enrich_context.emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class))) .thenReturn(Validation.success(_mapper.createObjectNode())); TestDedupEnrichmentModule test_module = new TestDedupEnrichmentModule(); final String ts_field = "@timestamp"; final ObjectNode old_json = _mapper.createObjectNode(); old_json.put("_id", "old_record"); old_json.put("@timestamp", 0L); old_json.put("url", "test"); final ObjectNode new_json = _mapper.createObjectNode(); new_json.put("@timestamp", 1L); new_json.put("url", "test"); final ObjectNode new_json_but_same_time = _mapper.createObjectNode(); new_json_but_same_time.put("@timestamp", 0L); new_json_but_same_time.put("url", "test"); Tuple3<Long, IBatchRecord, ObjectNode> new_record = Tuples._3T(0L, new BatchRecordUtils.JsonBatchRecord(new_json), _mapper.createObjectNode()); Tuple3<Long, IBatchRecord, ObjectNode> new_record_but_same_time = Tuples._3T(0L, new BatchRecordUtils.JsonBatchRecord(new_json_but_same_time), _mapper.createObjectNode()); new_record._2().getContent(); //(code coverage!) final TextNode key = new TextNode("url"); LinkedHashMap<JsonNode, LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>>> mutable_obj_map = new LinkedHashMap<>(); final LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>> new_records = Stream.of(new_record) .collect(Collectors.toCollection(LinkedList::new)); final LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>> new_records_but_same_time = Stream .of(new_record_but_same_time).collect(Collectors.toCollection(LinkedList::new)); // Simple case Leave policy { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.leave).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(0, _called_batch.get()); // No annotations/mutations assertEquals("{}", new_record._3().toString()); // Object removed from mutable map assertEquals(1, mutable_obj_map.size()); } // Simple case update policy - time updates final Consumer<Boolean> test_time_updates = delete_unhandled -> { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.update) .with(DocumentSchemaBean::delete_unhandled_duplicates, delete_unhandled).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); // (add the same object twice to test the "return ids to delete" functionality) final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records, Arrays.asList(old_json, old_json), key, mutable_obj_map); if (delete_unhandled) { assertEquals(Arrays.asList("old_record"), ret_val.sorted() .map(j -> DeduplicationService.jsonToObject(j)).collect(Collectors.toList())); } else { assertEquals(0L, ret_val.count()); } // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(0, _called_batch.get()); // _id assertEquals("{\"_id\":\"old_record\"}", new_record._3().toString()); // Object removed from mutable map assertEquals(2, mutable_obj_map.size()); }; test_time_updates.accept(true); test_time_updates.accept(false); // Simple case update policy - times the same { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.update) .with(DocumentSchemaBean::delete_unhandled_duplicates, false).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records_but_same_time, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(0, _called_batch.get()); // No annotations/mutations assertEquals("{}", new_record_but_same_time._3().toString()); // Object removed from mutable map assertEquals(1, mutable_obj_map.size()); } // overwrite final Consumer<Boolean> test_overwrites = delete_unhandled -> { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.overwrite) .with(DocumentSchemaBean::delete_unhandled_duplicates, delete_unhandled).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records, Arrays.asList(old_json, old_json), key, mutable_obj_map); if (delete_unhandled) { assertEquals(Arrays.asList("old_record"), ret_val.sorted() .map(j -> DeduplicationService.jsonToObject(j)).collect(Collectors.toList())); } else { assertEquals(0L, ret_val.count()); } // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(0, _called_batch.get()); // _id assertEquals("{\"_id\":\"old_record\"}", new_record._3().toString()); // Object removed from mutable map assertEquals(2, mutable_obj_map.size()); }; test_overwrites.accept(true); test_overwrites.accept(false); //(check ignores times) { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.overwrite) .with(DocumentSchemaBean::delete_unhandled_duplicates, false).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records_but_same_time, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(0, _called_batch.get()); // _id assertEquals("{\"_id\":\"old_record\"}", new_record_but_same_time._3().toString()); // Object removed from mutable map assertEquals(2, mutable_obj_map.size()); } // custom { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.custom) .with(DocumentSchemaBean::delete_unhandled_duplicates, false).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(2, _called_batch.get()); //(old + new) // _id assertEquals("{}", new_record._3().toString()); // up to the custom code to do this // Object removed from mutable map assertEquals(1, mutable_obj_map.size()); //(remove since it's the responsibility of the custom code to emit) } //(check ignores times) { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.custom) .with(DocumentSchemaBean::delete_unhandled_duplicates, false).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records_but_same_time, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(2, _called_batch.get()); //(old + new) // _id assertEquals("{}", new_record_but_same_time._3().toString()); // up to the custom code to do this // Object removed from mutable map assertEquals(1, mutable_obj_map.size()); //(remove since it's the responsibility of the custom code to emit) } // Simple case *custom* update policy - time updates { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.custom_update) .with(DocumentSchemaBean::delete_unhandled_duplicates, false).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(2, _called_batch.get()); //(old + new) // _id assertEquals("{}", new_record._3().toString()); // up to the custom code to do this // Object removed from mutable map assertEquals(1, mutable_obj_map.size()); //(remove since it's the responsibility of the custom code to emit) } // Simple case *custom* update policy - times the same { //(reset) mutable_obj_map.clear(); mutable_obj_map.put(new TextNode("never_changed"), new_records); mutable_obj_map.put(new TextNode("url"), new_records); assertEquals(2, mutable_obj_map.size()); new_record._3().removeAll(); new_record_but_same_time._3().removeAll(); _called_batch.set(0); DocumentSchemaBean config = BeanTemplateUtils.build(DocumentSchemaBean.class) .with(DocumentSchemaBean::deduplication_policy, DeduplicationPolicy.custom_update) .with(DocumentSchemaBean::delete_unhandled_duplicates, false).done().get(); DeduplicationEnrichmentContext test_context = new DeduplicationEnrichmentContext(enrich_context, config, j -> Optional.empty()); final Stream<JsonNode> ret_val = DeduplicationService.handleDuplicateRecord(config, Optional.of(Tuples._2T(test_module, test_context)), ts_field, new_records_but_same_time, Arrays.asList(old_json), key, mutable_obj_map); assertEquals(0L, ret_val.count()); // Nothing emitted Mockito.verify(enrich_context, Mockito.times(0)).emitImmutableObject(Mockito.any(Long.class), Mockito.any(JsonNode.class), Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class)); // No custom processing performed assertEquals(0, _called_batch.get()); // No annotations/mutations assertEquals("{}", new_record_but_same_time._3().toString()); // Object removed from mutable map assertEquals(1, mutable_obj_map.size()); } }