Example usage for org.springframework.kafka.listener KafkaMessageListenerContainer KafkaMessageListenerContainer

List of usage examples for org.springframework.kafka.listener KafkaMessageListenerContainer KafkaMessageListenerContainer

  1. HOME
  2. Java
  3. org.springframework
  4. org.springframework.kafka.listener.*
  5. KafkaMessageListenerContainer
  6. KafkaMessageListenerContainer

Introduction

In this page you can find the example usage for org.springframework.kafka.listener KafkaMessageListenerContainer KafkaMessageListenerContainer.

Prototype

public KafkaMessageListenerContainer(ConsumerFactory<? super K, ? super V> consumerFactory,
        ContainerProperties containerProperties)

Source Link

Document

Construct an instance with the supplied configuration properties.

Usage

From source file:org.s1p.app8.S1pKafkaApplication.java

@Bean
public KafkaMessageListenerContainer<String, String> container(ConsumerFactory<String, String> consumerFactory,
        ConfigProperties config) {//from  w w  w  .  jav a  2 s . c o  m
    ContainerProperties containerProperties = new ContainerProperties(config.getTopic());
    containerProperties.setMessageListener(listener());
    return new KafkaMessageListenerContainer<>(consumerFactory, containerProperties);
}

From source file:org.s1p.app9.S1pKafkaApplication.java

@Bean
public KafkaMessageListenerContainer<String, String> container(ConsumerFactory<String, String> consumerFactory,
        ConfigProperties config) {// w ww  . j av a 2 s .  c om
    ContainerProperties containerProperties = new ContainerProperties(config.getTopic());
    containerProperties.setMessageListener(listener());
    containerProperties.setAckMode(AckMode.MANUAL_IMMEDIATE);
    return new KafkaMessageListenerContainer<>(consumerFactory, containerProperties);
}

From source file:io.pivotal.cf.service.connector.KafkaRepository.java

private KafkaMessageListenerContainer<Integer, String> createContainer(ContainerProperties containerProps) {
    Map<String, Object> props = consumerProps();
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);//from   w  w w  . j a  va2s  .c o m
    return container;
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

@Test
public void testSlowListener() throws Exception {
    logger.info("Start " + this.testName.getMethodName());
    Map<String, Object> props = KafkaTestUtils.consumerProps("slow1", "false", embeddedKafka);
    //      props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, 6); // 2 per poll
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
    ContainerProperties containerProps = new ContainerProperties(topic1);

    final CountDownLatch latch = new CountDownLatch(6);
    final BitSet bitSet = new BitSet(6);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        logger.info("slow1: " + message);
        bitSet.set((int) (message.partition() * 3 + message.offset()));
        try {/*  w w w .  j a  v a2 s  . com*/
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
        latch.countDown();
    });
    containerProps.setPauseAfter(100);
    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);
    container.setBeanName("testSlow1");

    container.start();
    Consumer<?, ?> consumer = spyOnConsumer(container);
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<Integer, String>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic1);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    Thread.sleep(300);
    template.sendDefault(0, "fiz");
    template.sendDefault(2, "buz");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(bitSet.cardinality()).isEqualTo(6);
    verify(consumer, atLeastOnce()).pause(anyObject());
    verify(consumer, atLeastOnce()).resume(anyObject());
    container.stop();
    logger.info("Stop " + this.testName.getMethodName());
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

private void testSlowListenerManualGuts(AckMode ackMode, String topic) throws Exception {
    logger.info("Start " + this.testName.getMethodName() + ackMode);
    Map<String, Object> props = KafkaTestUtils.consumerProps("slow2", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
    ContainerProperties containerProps = new ContainerProperties(topic);
    containerProps.setSyncCommits(true);

    final CountDownLatch latch = new CountDownLatch(6);
    final BitSet bitSet = new BitSet(4);
    containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> {
        logger.info("slow2: " + message);
        bitSet.set((int) (message.partition() * 3 + message.offset()));
        try {/*  www.  ja v a2  s .  c  o m*/
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
        ack.acknowledge();
        latch.countDown();
    });
    containerProps.setPauseAfter(100);
    containerProps.setAckMode(ackMode);

    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);
    container.setBeanName("testSlow2");
    container.start();

    Consumer<?, ?> consumer = spyOnConsumer(container);

    final CountDownLatch commitLatch = new CountDownLatch(7);

    willAnswer(invocation -> {

        try {
            return invocation.callRealMethod();
        } finally {
            commitLatch.countDown();
        }

    }).given(consumer).commitSync(any());

    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<Integer, String>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    Thread.sleep(300);
    template.sendDefault(0, "fiz");
    template.sendDefault(2, "buz");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(commitLatch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(bitSet.cardinality()).isEqualTo(6);
    verify(consumer, atLeastOnce()).pause(anyObject());
    verify(consumer, atLeastOnce()).resume(anyObject());
    container.stop();
    logger.info("Stop " + this.testName.getMethodName() + ackMode);
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

@Test
public void testSlowConsumerCommitsAreProcessed() throws Exception {
    Map<String, Object> props = KafkaTestUtils.consumerProps("slow", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic5);
    containerProps.setAckCount(1);//ww w  .j ava  2 s .c o  m
    containerProps.setPauseAfter(100);
    containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE);
    containerProps.setSyncCommits(true);

    containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> {
        logger.info("slow: " + message);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
        ack.acknowledge();
    });

    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);

    container.setBeanName("testSlow");

    container.start();
    Consumer<?, ?> consumer = spyOnConsumer(container);

    final CountDownLatch latch = new CountDownLatch(3);

    willAnswer(invocation -> {

        try {
            return invocation.callRealMethod();
        } finally {
            latch.countDown();
        }

    }).given(consumer).commitSync(any());

    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic5);
    template.sendDefault(0, 0, "foo");
    template.sendDefault(1, 2, "bar");
    template.flush();
    Thread.sleep(300);
    template.sendDefault(0, 0, "fiz");
    template.sendDefault(1, 2, "buz");
    template.flush();

    // Verify that commitSync is called when paused
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    verify(consumer, atLeastOnce()).pause(anyObject());
    verify(consumer, atLeastOnce()).resume(anyObject());
    container.stop();
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

@Test
public void testCommitsAreFlushedOnStop() throws Exception {
    Map<String, Object> props = KafkaTestUtils.consumerProps("flushedOnStop", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic5);
    containerProps.setAckCount(1);/*  w w w  . j  av a 2 s. c  om*/
    containerProps.setPauseAfter(100);
    // set large values, ensuring that commits don't happen before `stop()`
    containerProps.setAckTime(20000);
    containerProps.setAckCount(20000);
    containerProps.setAckMode(AckMode.COUNT_TIME);

    final CountDownLatch latch = new CountDownLatch(4);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        logger.info("flushed: " + message);
        latch.countDown();
    });
    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);
    container.setBeanName("testManualFlushed");

    container.start();
    Consumer<?, ?> consumer = spyOnConsumer(container);
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic5);
    template.sendDefault(0, 0, "foo");
    template.sendDefault(1, 2, "bar");
    template.flush();
    Thread.sleep(300);
    template.sendDefault(0, 0, "fiz");
    template.sendDefault(1, 2, "buz");
    template.flush();

    // Verify that commitSync is called when paused
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    // Verify that just the initial commit is processed before stop
    verify(consumer, times(1)).commitSync(any());
    container.stop();
    // Verify that a commit has been made on stop
    verify(consumer, times(2)).commitSync(any());
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

@Test
public void testSlowConsumerWithException() throws Exception {
    logger.info("Start " + this.testName.getMethodName());
    Map<String, Object> props = KafkaTestUtils.consumerProps("slow3", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
    ContainerProperties containerProps = new ContainerProperties(topic3);

    final CountDownLatch latch = new CountDownLatch(18);
    final BitSet bitSet = new BitSet(6);
    final Map<String, AtomicInteger> faults = new HashMap<>();
    RetryingMessageListenerAdapter<Integer, String> adapter = new RetryingMessageListenerAdapter<>(
            new MessageListener<Integer, String>() {

                @Override//from  www .  ja  v a  2 s .  co m
                public void onMessage(ConsumerRecord<Integer, String> message) {
                    logger.info("slow3: " + message);
                    bitSet.set((int) (message.partition() * 3 + message.offset()));
                    String key = message.topic() + message.partition() + message.offset();
                    if (faults.get(key) == null) {
                        faults.put(key, new AtomicInteger(1));
                    } else {
                        faults.get(key).incrementAndGet();
                    }
                    latch.countDown(); // 3 per = 18
                    if (faults.get(key).get() < 3) { // succeed on the third attempt
                        throw new FooEx();
                    }
                }

            }, buildRetry(), null);
    containerProps.setMessageListener(adapter);
    containerProps.setPauseAfter(100);

    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);
    container.setBeanName("testSlow3");

    container.start();
    Consumer<?, ?> consumer = spyOnConsumer(container);
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic3);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    Thread.sleep(300);
    template.sendDefault(0, "fiz");
    template.sendDefault(2, "buz");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(bitSet.cardinality()).isEqualTo(6);
    verify(consumer, atLeastOnce()).pause(anyObject());
    verify(consumer, atLeastOnce()).resume(anyObject());
    container.stop();
    logger.info("Stop " + this.testName.getMethodName());
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

@Test
public void testSlowConsumerWithSlowThenExceptionThenGood() throws Exception {
    logger.info("Start " + this.testName.getMethodName());
    Map<String, Object> props = KafkaTestUtils.consumerProps("slow4", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
    ContainerProperties containerProps = new ContainerProperties(topic4);
    final CountDownLatch latch = new CountDownLatch(18);
    final BitSet bitSet = new BitSet(6);
    final Map<String, AtomicInteger> faults = new HashMap<>();
    RetryingMessageListenerAdapter<Integer, String> adapter = new RetryingMessageListenerAdapter<>(
            new MessageListener<Integer, String>() {

                @Override/* w  w  w .j av a  2s  .  c o  m*/
                public void onMessage(ConsumerRecord<Integer, String> message) {
                    logger.info("slow4: " + message);
                    bitSet.set((int) (message.partition() * 4 + message.offset()));
                    String key = message.topic() + message.partition() + message.offset();
                    if (faults.get(key) == null) {
                        faults.put(key, new AtomicInteger(1));
                    } else {
                        faults.get(key).incrementAndGet();
                    }
                    latch.countDown(); // 3 per = 18
                    if (faults.get(key).get() == 1) {
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            Thread.currentThread().interrupt();
                        }
                    }
                    if (faults.get(key).get() < 3) { // succeed on the third attempt
                        throw new FooEx();
                    }
                }

            }, buildRetry(), null);
    containerProps.setMessageListener(adapter);
    containerProps.setPauseAfter(100);

    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);
    container.setBeanName("testSlow4");

    container.start();
    Consumer<?, ?> consumer = spyOnConsumer(container);
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic4);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    Thread.sleep(300);
    template.sendDefault(0, "fiz");
    template.sendDefault(2, "buz");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(bitSet.cardinality()).isEqualTo(6);
    verify(consumer, atLeastOnce()).pause(anyObject());
    verify(consumer, atLeastOnce()).resume(anyObject());
    container.stop();
    logger.info("Stop " + this.testName.getMethodName());
}

From source file:org.springframework.kafka.listener.KafkaMessageListenerContainerTests.java

@Test
public void testRecordAck() throws Exception {
    logger.info("Start record ack");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test6", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic6);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        logger.info("record ack: " + message);
    });//from  w  ww .  ja v a  2s  . c  o m
    containerProps.setSyncCommits(true);
    containerProps.setAckMode(AckMode.RECORD);
    containerProps.setAckOnError(false);

    KafkaMessageListenerContainer<Integer, String> container = new KafkaMessageListenerContainer<>(cf,
            containerProps);
    container.setBeanName("testRecordAcks");
    container.start();
    Consumer<?, ?> containerConsumer = spyOnConsumer(container);
    final CountDownLatch latch = new CountDownLatch(2);
    willAnswer(invocation -> {

        @SuppressWarnings({ "unchecked" })
        Map<TopicPartition, OffsetAndMetadata> map = (Map<TopicPartition, OffsetAndMetadata>) invocation
                .getArguments()[0];
        try {
            return invocation.callRealMethod();
        } finally {
            for (Entry<TopicPartition, OffsetAndMetadata> entry : map.entrySet()) {
                if (entry.getValue().offset() == 2) {
                    latch.countDown();
                }
            }
        }

    }).given(containerConsumer).commitSync(any());
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic6);
    template.sendDefault(0, 0, "foo");
    template.sendDefault(1, 0, "bar");
    template.sendDefault(0, 0, "baz");
    template.sendDefault(1, 0, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    Consumer<Integer, String> consumer = cf.createConsumer();
    consumer.assign(Arrays.asList(new TopicPartition(topic6, 0), new TopicPartition(topic6, 1)));
    assertThat(consumer.position(new TopicPartition(topic6, 0))).isEqualTo(2);
    assertThat(consumer.position(new TopicPartition(topic6, 1))).isEqualTo(2);
    container.stop();
    consumer.close();
    logger.info("Stop record ack");
}