Example usage for org.springframework.kafka.core KafkaTemplate flush

List of usage examples for org.springframework.kafka.core KafkaTemplate flush

  1. HOME
  2. Java
  3. org.springframework
  4. org.springframework.kafka.core.*
  5. KafkaTemplate
  6. flush

Introduction

In this page you can find the example usage for org.springframework.kafka.core KafkaTemplate flush.

Prototype

@Override
public void flush()

Source Link

Document

Note It only makes sense to invoke this method if the ProducerFactory serves up a singleton producer (such as the DefaultKafkaProducerFactory ).

Usage

From source file:io.pivotal.cf.service.connector.KafkaRepository.java

public ListenableFuture<SendResult<Integer, String>> sendMessage(String message)
        throws ExecutionException, InterruptedException {
    KafkaTemplate<Integer, String> template = getTemplate();
    ListenableFuture<SendResult<Integer, String>> future = template.send(info.getTopicName(), message);
    template.flush();
    return future;
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testAutoCommit() throws Exception {
    this.logger.info("Start auto");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test1", "true", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic1);

    final CountDownLatch latch = new CountDownLatch(4);
    final Set<String> listenerThreadNames = new ConcurrentSkipListSet<>();
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("auto: " + message);
        listenerThreadNames.add(Thread.currentThread().getName());
        latch.countDown();//from www. j  a v  a  2s  . c  o  m
    });

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(2);
    container.setBeanName("testAuto");
    container.start();

    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic1);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(listenerThreadNames).allMatch(threadName -> threadName.contains("-consumer-"));
    @SuppressWarnings("unchecked")
    List<KafkaMessageListenerContainer<Integer, String>> containers = KafkaTestUtils.getPropertyValue(container,
            "containers", List.class);
    assertThat(containers.size()).isEqualTo(2);
    for (int i = 0; i < 2; i++) {
        assertThat(KafkaTestUtils.getPropertyValue(containers.get(i), "listenerConsumer.acks", Collection.class)
                .size()).isEqualTo(0);
    }
    container.stop();
    this.logger.info("Stop auto");
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testAutoCommitWithRebalanceListener() throws Exception {
    this.logger.info("Start auto");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test10", "true", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic1);

    final CountDownLatch latch = new CountDownLatch(4);
    final Set<String> listenerThreadNames = new ConcurrentSkipListSet<>();
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("auto: " + message);
        listenerThreadNames.add(Thread.currentThread().getName());
        latch.countDown();// ww w.  j a  v a  2s .co  m
    });
    final CountDownLatch rebalancePartitionsAssignedLatch = new CountDownLatch(2);
    final CountDownLatch rebalancePartitionsRevokedLatch = new CountDownLatch(2);
    containerProps.setConsumerRebalanceListener(new ConsumerRebalanceListener() {

        @Override
        public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
            ConcurrentMessageListenerContainerTests.this.logger
                    .info("In test, partitions revoked:" + partitions);
            rebalancePartitionsRevokedLatch.countDown();
        }

        @Override
        public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
            ConcurrentMessageListenerContainerTests.this.logger
                    .info("In test, partitions assigned:" + partitions);
            rebalancePartitionsAssignedLatch.countDown();
        }

    });

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(2);
    container.setBeanName("testAuto");
    container.start();

    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic1);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(rebalancePartitionsAssignedLatch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(rebalancePartitionsRevokedLatch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(listenerThreadNames).allMatch(threadName -> threadName.contains("-consumer-"));
    container.stop();
    this.logger.info("Stop auto");
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testAfterListenCommit() throws Exception {
    this.logger.info("Start manual");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test2", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic2);

    final CountDownLatch latch = new CountDownLatch(4);
    final Set<String> listenerThreadNames = new ConcurrentSkipListSet<>();
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("manual: " + message);
        listenerThreadNames.add(Thread.currentThread().getName());
        latch.countDown();//from  w  w  w .jav  a 2  s.com
    });

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(2);
    container.setBeanName("testBatch");
    container.start();

    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic2);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(listenerThreadNames).allMatch(threadName -> threadName.contains("-listener-"));
    container.stop();
    this.logger.info("Stop manual");
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

private void testManualCommitGuts(AckMode ackMode, String topic) throws Exception {
    this.logger.info("Start " + ackMode);
    Map<String, Object> props = KafkaTestUtils.consumerProps("test" + ackMode, "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic);
    final CountDownLatch latch = new CountDownLatch(4);
    containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("manual: " + message);
        ack.acknowledge();/*from w w  w .  j av a2  s  . c  o m*/
        latch.countDown();
    });

    containerProps.setAckMode(ackMode);
    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(2);
    container.setBeanName("test" + ackMode);
    container.start();

    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());

    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    container.stop();
    this.logger.info("Stop " + ackMode);
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
@Ignore // TODO https://github.com/spring-projects/spring-kafka/issues/62 using SYNC for avoidance
public void testManualCommitExisting() throws Exception {
    this.logger.info("Start MANUAL_IMMEDIATE with Existing");
    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic7);//from  ww w.j a v  a  2  s.c o m
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    Map<String, Object> props = KafkaTestUtils.consumerProps("testManualExisting", "false", embeddedKafka);
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic7);
    final CountDownLatch latch = new CountDownLatch(8);
    containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("manualExisting: " + message);
        ack.acknowledge();
        latch.countDown();
    });
    containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE);

    final CountDownLatch commits = new CountDownLatch(8);
    final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
    containerProps.setCommitCallback((offsets, exception) -> {
        commits.countDown();
        if (exception != null) {
            exceptionRef.compareAndSet(null, exception);
        }
    });

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(1);
    container.setBeanName("testManualExisting");

    container.start();
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
    template.sendDefault(0, "fooo");
    template.sendDefault(2, "barr");
    template.sendDefault(0, "bazz");
    template.sendDefault(2, "quxx");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(commits.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(exceptionRef.get()).isNull();
    container.stop();
    this.logger.info("Stop MANUAL_IMMEDIATE with Existing");
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testManualCommitSyncExisting() throws Exception {
    this.logger.info("Start MANUAL_IMMEDIATE with Existing");
    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<Integer, String>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic8);//w ww. j a v a  2s . c  om
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    Map<String, Object> props = KafkaTestUtils.consumerProps("testManualExistingSync", "false", embeddedKafka);
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<Integer, String>(props);
    ContainerProperties containerProps = new ContainerProperties(topic8);
    containerProps.setSyncCommits(true);
    final CountDownLatch latch = new CountDownLatch(8);
    final BitSet bitSet = new BitSet(8);
    containerProps.setMessageListener((AcknowledgingMessageListener<Integer, String>) (message, ack) -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("manualExisting: " + message);
        ack.acknowledge();
        bitSet.set((int) (message.partition() * 4 + message.offset()));
        latch.countDown();
    });
    containerProps.setAckMode(AckMode.MANUAL_IMMEDIATE);

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(1);
    container.setBeanName("testManualExisting");
    container.start();
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
    template.sendDefault(0, "fooo");
    template.sendDefault(2, "barr");
    template.sendDefault(0, "bazz");
    template.sendDefault(2, "quxx");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(bitSet.cardinality()).isEqualTo(8);
    container.stop();
    this.logger.info("Stop MANUAL_IMMEDIATE with Existing");
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testListenerException() throws Exception {
    this.logger.info("Start exception");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test1", "true", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic6);
    containerProps.setAckCount(23);/*w ww. ja v  a 2s .  co m*/
    ContainerProperties containerProps2 = new ContainerProperties(topic2);
    BeanUtils.copyProperties(containerProps, containerProps2, "topics", "topicPartitions", "topicPattern",
            "ackCount", "ackTime");
    final CountDownLatch latch = new CountDownLatch(4);
    final AtomicBoolean catchError = new AtomicBoolean(false);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        ConcurrentMessageListenerContainerTests.this.logger.info("auto: " + message);
        latch.countDown();
        throw new RuntimeException("intended");
    });
    containerProps.setGenericErrorHandler((ErrorHandler) (thrownException, record) -> catchError.set(true));

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(2);
    container.setBeanName("testException");

    container.start();
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic6);
    template.sendDefault(0, "foo");
    template.sendDefault(2, "bar");
    template.sendDefault(0, "baz");
    template.sendDefault(2, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(catchError.get()).isTrue();
    container.stop();
    this.logger.info("Stop exception");

}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testAckOnErrorRecord() throws Exception {
    logger.info("Start ack on error");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test9", "false", embeddedKafka);
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    final CountDownLatch latch = new CountDownLatch(4);
    ContainerProperties containerProps = new ContainerProperties(topic9);
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        logger.info("auto ack on error: " + message);
        latch.countDown();/*from w  ww.j a  v  a 2s.  co m*/
        if (message.value().startsWith("b")) {
            throw new RuntimeException();
        }
    });
    containerProps.setSyncCommits(true);
    containerProps.setAckMode(AckMode.RECORD);
    containerProps.setAckOnError(false);
    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    container.setConcurrency(2);
    container.setBeanName("testAckOnError");
    container.start();
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic9);
    template.sendDefault(0, 0, "foo");
    template.sendDefault(1, 0, "bar");
    template.sendDefault(0, 0, "baz");
    template.sendDefault(1, 0, "qux");
    template.flush();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    container.stop();
    Consumer<Integer, String> consumer = cf.createConsumer();
    consumer.assign(Arrays.asList(new TopicPartition(topic9, 0), new TopicPartition(topic9, 1)));
    // this consumer is positioned at 1, the next offset after the successfully
    // processed 'foo'
    // it has not been updated because 'bar' failed
    assertThat(consumer.position(new TopicPartition(topic9, 0))).isEqualTo(1);
    // this consumer is positioned at 1, the next offset after the successfully
    // processed 'qux'
    // it has been updated even 'baz' failed
    assertThat(consumer.position(new TopicPartition(topic9, 1))).isEqualTo(2);
    consumer.close();
    logger.info("Stop ack on error");
}

From source file:org.springframework.kafka.listener.ConcurrentMessageListenerContainerTests.java

@Test
public void testRebalanceWithSlowConsumer() throws Exception {
    this.logger.info("Start auto");
    Map<String, Object> props = KafkaTestUtils.consumerProps("test101", "false", embeddedKafka);
    props.put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, "20000");
    DefaultKafkaConsumerFactory<Integer, String> cf = new DefaultKafkaConsumerFactory<>(props);
    ContainerProperties containerProps = new ContainerProperties(topic1);
    final CountDownLatch latch = new CountDownLatch(8);
    final Set<String> listenerThreadNames = Collections.synchronizedSet(new HashSet<String>());
    List<String> receivedMessages = Collections.synchronizedList(new ArrayList<>());
    containerProps.setMessageListener((MessageListener<Integer, String>) message -> {
        listenerThreadNames.add(Thread.currentThread().getName());
        try {/*from  w  w w.  ja v  a 2  s  .  c  om*/
            Thread.sleep(2000);
        } catch (InterruptedException e) {
            // ignore
        }
        receivedMessages.add(message.value());
        listenerThreadNames.add(Thread.currentThread().getName());
        latch.countDown();
    });

    ConcurrentMessageListenerContainer<Integer, String> container = new ConcurrentMessageListenerContainer<>(cf,
            containerProps);
    ConcurrentMessageListenerContainer<Integer, String> container2 = new ConcurrentMessageListenerContainer<>(
            cf, containerProps);
    container.setConcurrency(1);
    container2.setConcurrency(1);
    container.setBeanName("testAuto");
    container2.setBeanName("testAuto2");
    container.start();
    ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic());
    Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
    ProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
    KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf);
    template.setDefaultTopic(topic1);
    template.sendDefault(0, 0, "foo");
    template.sendDefault(0, 2, "bar");
    template.sendDefault(0, 0, "baz");
    template.sendDefault(0, 2, "qux");
    template.sendDefault(1, 2, "corge");
    template.sendDefault(1, 2, "grault");
    template.sendDefault(1, 2, "garply");
    template.sendDefault(1, 2, "waldo");
    template.flush();
    container2.start();
    assertThat(latch.await(60, TimeUnit.SECONDS)).isTrue();
    assertThat(receivedMessages).containsOnlyOnce("foo", "bar", "baz", "qux", "corge", "grault", "garply",
            "waldo");
    // all messages are received
    assertThat(receivedMessages).hasSize(8);
    // messages are received on separate threads
    assertThat(listenerThreadNames.size()).isGreaterThanOrEqualTo(2);
    container.stop();
    container2.stop();
    this.logger.info("Stop auto");
}