List of usage examples for org.springframework.kafka.core DefaultKafkaProducerFactory DefaultKafkaProducerFactory
public DefaultKafkaProducerFactory(Map<String, Object> configs)
From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java
@Test public void testListenerException() throws Exception { this.logger.info("Start exception"); Map<String, Object> props = KafkaTestUtils.consumerProps("test1", "true", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props); ContainerProperties containerProps = new ContainerProperties(topic6); containerProps.setAckCount(23);//from w ww .j a v a 2 s.c o m ContainerProperties containerProps2 = new ContainerProperties(topic2); BeanUtils.copyProperties(containerProps, containerProps2, "topics", "topicPartitions", "topicPattern", "ackCount", "ackTime"); final CountDownLatch latch = new CountDownLatch(4); final AtomicBoolean catchError = new AtomicBoolean(false); containerProps.setMessageListener((MessageListener<Integer, String>) message -> { ConcurrentMessageListenerContainerTests.this.logger.info("auto: " + message); latch.countDown(); throw new RuntimeException("intended"); }); containerProps.setGenericErrorHandler((ErrorHandler) (thrownException, record) -> catchError.set(true)); ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf, containerProps); container.setConcurrency(2); container.setBeanName("testException"); container.start(); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic6); template.sendDefault(0, "foo"); template.sendDefault(2, "bar"); template.sendDefault(0, "baz"); template.sendDefault(2, "qux"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(catchError.get()).isTrue(); container.stop(); this.logger.info("Stop exception"); }
From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java
@Test public void testAckOnErrorRecord() throws Exception { logger.info("Start ack on error"); Map<String, Object> props = KafkaTestUtils.consumerProps("test9", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props); final CountDownLatch latch = new CountDownLatch(4); ContainerProperties containerProps = new ContainerProperties(topic9); containerProps.setMessageListener((MessageListener<Integer, String>) message -> { logger.info("auto ack on error: " + message); latch.countDown();/*from www.j av a2 s .com*/ if (message.value().startsWith("b")) { throw new RuntimeException(); } }); containerProps.setSyncCommits(true); containerProps.setAckMode(AckMode.RECORD); containerProps.setAckOnError(false); ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf, containerProps); container.setConcurrency(2); container.setBeanName("testAckOnError"); container.start(); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic9); template.sendDefault(0, 0, "foo"); template.sendDefault(1, 0, "bar"); template.sendDefault(0, 0, "baz"); template.sendDefault(1, 0, "qux"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); container.stop(); Consumer<Integer, String> consumer = cf.createConsumer(); consumer.assign(Arrays.asList(new TopicPartition(topic9, 0), new TopicPartition(topic9, 1))); // this consumer is positioned at 1, the next offset after the successfully // processed 'foo' // it has not been updated because 'bar' failed assertThat(consumer.position(new TopicPartition(topic9, 0))).isEqualTo(1); // this consumer is positioned at 1, the next offset after the successfully // processed 'qux' // it has been updated even 'baz' failed assertThat(consumer.position(new TopicPartition(topic9, 1))).isEqualTo(2); consumer.close(); logger.info("Stop ack on error"); }
From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java
@Test public void testRebalanceWithSlowConsumer() throws Exception { this.logger.info("Start auto"); Map<String, Object> props = KafkaTestUtils.consumerProps("test101", "false", embeddedKafka); props.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, "20000"); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props); ContainerProperties containerProps = new ContainerProperties(topic1); final CountDownLatch latch = new CountDownLatch(8); final Set<String> listenerThreadNames = Collections.synchronizedSet(new HashSet<String>()); List<String> receivedMessages = Collections.synchronizedList(new ArrayList<>()); containerProps.setMessageListener((MessageListener<Integer, String>) message -> { listenerThreadNames.add(Thread.currentThread().getName()); try {//from w ww .jav a 2 s . co m Thread.sleep(2000); } catch (InterruptedException e) { // ignore } receivedMessages.add(message.value()); listenerThreadNames.add(Thread.currentThread().getName()); latch.countDown(); }); ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf, containerProps); ConcurrentMessageListenerContainer<Integer, String> container2 = new ConcurrentMessageListenerContainer<>( cf, containerProps); container.setConcurrency(1); container2.setConcurrency(1); container.setBeanName("testAuto"); container2.setBeanName("testAuto2"); container.start(); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic1); template.sendDefault(0, 0, "foo"); template.sendDefault(0, 2, "bar"); template.sendDefault(0, 0, "baz"); template.sendDefault(0, 2, "qux"); template.sendDefault(1, 2, "corge"); template.sendDefault(1, 2, "grault"); template.sendDefault(1, 2, "garply"); template.sendDefault(1, 2, "waldo"); template.flush(); container2.start(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(receivedMessages).containsOnlyOnce("foo", "bar", "baz", "qux", "corge", "grault", "garply", "waldo"); // all messages are received assertThat(receivedMessages).hasSize(8); // messages are received on separate threads assertThat(listenerThreadNames.size()).isGreaterThanOrEqualTo(2); container.stop(); container2.stop(); this.logger.info("Stop auto"); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
@Test public void testSlowListener() throws Exception { logger.info("Start " + this.testName.getMethodName()); Map<String, Object> props = KafkaTestUtils.consumerProps("slow1", "false", embeddedKafka); // props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, 6); // 2 per poll DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props); ContainerProperties containerProps = new ContainerProperties(topic1); final CountDownLatch latch = new CountDownLatch(6); final BitSet bitSet = new BitSet(6); containerProps.setMessageListener((MessageListener<Integer, String>) message -> { logger.info("slow1: " + message); bitSet.set((int) (message.partition() * 3 + message.offset())); try {/*from ww w.j av a 2s.c o m*/ Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } latch.countDown(); }); containerProps.setPauseAfter(100); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testSlow1"); container.start(); Consumer<?, ?> consumer = spyOnConsumer(container); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<Integer, String>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic1); template.sendDefault(0, "foo"); template.sendDefault(2, "bar"); template.sendDefault(0, "baz"); template.sendDefault(2, "qux"); template.flush(); Thread.sleep(300); template.sendDefault(0, "fiz"); template.sendDefault(2, "buz"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(bitSet.cardinality()).isEqualTo(6); verify(consumer, atLeastOnce()).pause(anyObject()); verify(consumer, atLeastOnce()).resume(anyObject()); container.stop(); logger.info("Stop " + this.testName.getMethodName()); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
private void testSlowListenerManualGuts(AckMode ackMode, String topic) throws Exception { logger.info("Start " + this.testName.getMethodName() + ackMode); Map<String, Object> props = KafkaTestUtils.consumerProps("slow2", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props); ContainerProperties containerProps = new ContainerProperties(topic); containerProps.setSyncCommits(true); final CountDownLatch latch = new CountDownLatch(6); final BitSet bitSet = new BitSet(4); containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> { logger.info("slow2: " + message); bitSet.set((int) (message.partition() * 3 + message.offset())); try {/*ww w . java 2 s. co m*/ Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } ack.acknowledge(); latch.countDown(); }); containerProps.setPauseAfter(100); containerProps.setAckMode(ackMode); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testSlow2"); container.start(); Consumer<?, ?> consumer = spyOnConsumer(container); final CountDownLatch commitLatch = new CountDownLatch(7); willAnswer(invocation -> { try { return invocation.callRealMethod(); } finally { commitLatch.countDown(); } }).given(consumer).commitSync(any()); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<Integer, String>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic); template.sendDefault(0, "foo"); template.sendDefault(2, "bar"); template.sendDefault(0, "baz"); template.sendDefault(2, "qux"); template.flush(); Thread.sleep(300); template.sendDefault(0, "fiz"); template.sendDefault(2, "buz"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(commitLatch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(bitSet.cardinality()).isEqualTo(6); verify(consumer, atLeastOnce()).pause(anyObject()); verify(consumer, atLeastOnce()).resume(anyObject()); container.stop(); logger.info("Stop " + this.testName.getMethodName() + ackMode); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
@Test public void testSlowConsumerCommitsAreProcessed() throws Exception { Map<String, Object> props = KafkaTestUtils.consumerProps("slow", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props); ContainerProperties containerProps = new ContainerProperties(topic5); containerProps.setAckCount(1);// w w w . ja v a2 s . co m containerProps.setPauseAfter(100); containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE); containerProps.setSyncCommits(true); containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> { logger.info("slow: " + message); try { Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } ack.acknowledge(); }); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testSlow"); container.start(); Consumer<?, ?> consumer = spyOnConsumer(container); final CountDownLatch latch = new CountDownLatch(3); willAnswer(invocation -> { try { return invocation.callRealMethod(); } finally { latch.countDown(); } }).given(consumer).commitSync(any()); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic5); template.sendDefault(0, 0, "foo"); template.sendDefault(1, 2, "bar"); template.flush(); Thread.sleep(300); template.sendDefault(0, 0, "fiz"); template.sendDefault(1, 2, "buz"); template.flush(); // Verify that commitSync is called when paused assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); verify(consumer, atLeastOnce()).pause(anyObject()); verify(consumer, atLeastOnce()).resume(anyObject()); container.stop(); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
@Test public void testCommitsAreFlushedOnStop() throws Exception { Map<String, Object> props = KafkaTestUtils.consumerProps("flushedOnStop", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props); ContainerProperties containerProps = new ContainerProperties(topic5); containerProps.setAckCount(1);//from w w w . j a v a 2 s . co m containerProps.setPauseAfter(100); // set large values, ensuring that commits don't happen before `stop()` containerProps.setAckTime(20000); containerProps.setAckCount(20000); containerProps.setAckMode(AckMode.COUNT_TIME); final CountDownLatch latch = new CountDownLatch(4); containerProps.setMessageListener((MessageListener<Integer, String>) message -> { logger.info("flushed: " + message); latch.countDown(); }); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testManualFlushed"); container.start(); Consumer<?, ?> consumer = spyOnConsumer(container); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic5); template.sendDefault(0, 0, "foo"); template.sendDefault(1, 2, "bar"); template.flush(); Thread.sleep(300); template.sendDefault(0, 0, "fiz"); template.sendDefault(1, 2, "buz"); template.flush(); // Verify that commitSync is called when paused assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); // Verify that just the initial commit is processed before stop verify(consumer, times(1)).commitSync(any()); container.stop(); // Verify that a commit has been made on stop verify(consumer, times(2)).commitSync(any()); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
@Test public void testSlowConsumerWithException() throws Exception { logger.info("Start " + this.testName.getMethodName()); Map<String, Object> props = KafkaTestUtils.consumerProps("slow3", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props); ContainerProperties containerProps = new ContainerProperties(topic3); final CountDownLatch latch = new CountDownLatch(18); final BitSet bitSet = new BitSet(6); final Map<String, AtomicInteger> faults = new HashMap<>(); RetryingMessageListenerAdapter<Integer, String> adapter = new RetryingMessageListenerAdapter<>( new MessageListener<Integer, String>() { @Override/* ww w . j av a2s . c o m*/ public void onMessage(ConsumerRecord<Integer, String> message) { logger.info("slow3: " + message); bitSet.set((int) (message.partition() * 3 + message.offset())); String key = message.topic() + message.partition() + message.offset(); if (faults.get(key) == null) { faults.put(key, new AtomicInteger(1)); } else { faults.get(key).incrementAndGet(); } latch.countDown(); // 3 per = 18 if (faults.get(key).get() < 3) { // succeed on the third attempt throw new FooEx(); } } }, buildRetry(), null); containerProps.setMessageListener(adapter); containerProps.setPauseAfter(100); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testSlow3"); container.start(); Consumer<?, ?> consumer = spyOnConsumer(container); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic3); template.sendDefault(0, "foo"); template.sendDefault(2, "bar"); template.sendDefault(0, "baz"); template.sendDefault(2, "qux"); template.flush(); Thread.sleep(300); template.sendDefault(0, "fiz"); template.sendDefault(2, "buz"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(bitSet.cardinality()).isEqualTo(6); verify(consumer, atLeastOnce()).pause(anyObject()); verify(consumer, atLeastOnce()).resume(anyObject()); container.stop(); logger.info("Stop " + this.testName.getMethodName()); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
@Test public void testSlowConsumerWithSlowThenExceptionThenGood() throws Exception { logger.info("Start " + this.testName.getMethodName()); Map<String, Object> props = KafkaTestUtils.consumerProps("slow4", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props); ContainerProperties containerProps = new ContainerProperties(topic4); final CountDownLatch latch = new CountDownLatch(18); final BitSet bitSet = new BitSet(6); final Map<String, AtomicInteger> faults = new HashMap<>(); RetryingMessageListenerAdapter<Integer, String> adapter = new RetryingMessageListenerAdapter<>( new MessageListener<Integer, String>() { @Override/*from w ww . ja v a2s .c om*/ public void onMessage(ConsumerRecord<Integer, String> message) { logger.info("slow4: " + message); bitSet.set((int) (message.partition() * 4 + message.offset())); String key = message.topic() + message.partition() + message.offset(); if (faults.get(key) == null) { faults.put(key, new AtomicInteger(1)); } else { faults.get(key).incrementAndGet(); } latch.countDown(); // 3 per = 18 if (faults.get(key).get() == 1) { try { Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } if (faults.get(key).get() < 3) { // succeed on the third attempt throw new FooEx(); } } }, buildRetry(), null); containerProps.setMessageListener(adapter); containerProps.setPauseAfter(100); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testSlow4"); container.start(); Consumer<?, ?> consumer = spyOnConsumer(container); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic4); template.sendDefault(0, "foo"); template.sendDefault(2, "bar"); template.sendDefault(0, "baz"); template.sendDefault(2, "qux"); template.flush(); Thread.sleep(300); template.sendDefault(0, "fiz"); template.sendDefault(2, "buz"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); assertThat(bitSet.cardinality()).isEqualTo(6); verify(consumer, atLeastOnce()).pause(anyObject()); verify(consumer, atLeastOnce()).resume(anyObject()); container.stop(); logger.info("Stop " + this.testName.getMethodName()); }
From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java
@Test public void testRecordAck() throws Exception { logger.info("Start record ack"); Map<String, Object> props = KafkaTestUtils.consumerProps("test6", "false", embeddedKafka); DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props); ContainerProperties containerProps = new ContainerProperties(topic6); containerProps.setMessageListener((MessageListener<Integer, String>) message -> { logger.info("record ack: " + message); });//from w w w .j a v a 2 s .c o m containerProps.setSyncCommits(true); containerProps.setAckMode(AckMode.RECORD); containerProps.setAckOnError(false); KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf, containerProps); container.setBeanName("testRecordAcks"); container.start(); Consumer<?, ?> containerConsumer = spyOnConsumer(container); final CountDownLatch latch = new CountDownLatch(2); willAnswer(invocation -> { @SuppressWarnings({ "unchecked" }) Map<TopicPartition, OffsetAndMetadata> map = (Map<TopicPartition, OffsetAndMetadata>) invocation .getArguments()[0]; try { return invocation.callRealMethod(); } finally { for (Entry<TopicPartition, OffsetAndMetadata> entry : map.entrySet()) { if (entry.getValue().offset() == 2) { latch.countDown(); } } } }).given(containerConsumer).commitSync(any()); ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()); Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka); ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps); KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf); template.setDefaultTopic(topic6); template.sendDefault(0, 0, "foo"); template.sendDefault(1, 0, "bar"); template.sendDefault(0, 0, "baz"); template.sendDefault(1, 0, "qux"); template.flush(); assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue(); Consumer<Integer, String> consumer = cf.createConsumer(); consumer.assign(Arrays.asList(new TopicPartition(topic6, 0), new TopicPartition(topic6, 1))); assertThat(consumer.position(new TopicPartition(topic6, 0))).isEqualTo(2); assertThat(consumer.position(new TopicPartition(topic6, 1))).isEqualTo(2); container.stop(); consumer.close(); logger.info("Stop record ack"); }