增加 spring cloud stream kafka 示例

This commit is contained in:
YunaiV
2020-03-09 19:28:31 +08:00
parent e818bc3314
commit cf7c2ff109
23 changed files with 547 additions and 5 deletions

View File

@@ -34,4 +34,20 @@ public class Demo01Controller {
return mySource.demo01Output().send(springMessage);
}
@GetMapping("/send_tag")
public boolean sendTag() {
for (String tag : new String[]{"yunai", "yutou", "tudou"}) {
// 创建 Message
Demo01Message message = new Demo01Message()
.setId(new Random().nextInt());
// 创建 Spring Message 对象
Message<Demo01Message> springMessage = MessageBuilder.withPayload(message)
.setHeader("tag", tag) // 设置 Tag
.build();
// 发送消息
mySource.demo01Output().send(springMessage);
}
return true;
}
}

View File

@@ -22,14 +22,13 @@ public class Demo01Controller {
@Autowired
private MySource mySource;
@GetMapping("/send_partition")
@GetMapping("/send_orderly")
public boolean send() {
// 创建 Message
Demo01Message message = new Demo01Message()
.setId(new Random().nextInt());
// 创建 Spring Message 对象
Message<Demo01Message> springMessage = MessageBuilder.withPayload(message)
.setHeader("partitionKey", 1)
.build();
// 发送消息
return mySource.demo01Output().send(springMessage);

View File

@@ -22,10 +22,11 @@ spring:
destination: DEMO-TOPIC-03 # 目的地。这里使用 RabbitMQ Exchange
content-type: application/json # 内容格式。这里使用 JSON
binder: rabbit001 # 设置使用的 Binder 名字
# Producer 配置项,对应 ProducerProperties 类
producer:
partition-key-expression: headers['partitionKey']
partition-count: 4
required-groups: demo01-consumer-group-DEMO-TOPIC-01 #only applicable for rabbit
partition-key-expression: payload['id'] # 分区 key 表达式。该表达式基于 Spring EL从消息中获得分区 key。
partition-count: 2 # 分区大小,默认为 1 分区
# required-groups: demo01-consumer-group-DEMO-TOPIC-01 #only applicable for rabbit
server:
port: 18080

View File

@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>labx-11</artifactId>
<groupId>cn.iocoder.springboot.labs</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>labx-11-sc-stream-kafka-consumer-filter</artifactId>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring.boot.version>2.2.4.RELEASE</spring.boot.version>
<spring.cloud.version>Hoxton.SR1</spring.cloud.version>
</properties>
<!--
引入 Spring Boot、Spring Cloud、Spring Cloud Alibaba 三者 BOM 文件,进行依赖版本的管理,防止不兼容。
在 https://dwz.cn/mcLIfNKt 文章中Spring Cloud Alibaba 开发团队推荐了三者的依赖关系
-->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>${spring.boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring.cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- 引入 SpringMVC 相关依赖,并实现对其的自动配置 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- 引入 Spring Cloud Stream Kafka 相关依赖,将 Kafka 作为消息队列,并实现对其的自动配置 -->
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,16 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo;
import cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.listener.MySink;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
@SpringBootApplication
@EnableBinding(MySink.class)
public class ConsumerApplication {
public static void main(String[] args) {
SpringApplication.run(ConsumerApplication.class, args);
}
}

View File

@@ -0,0 +1,20 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.listener;
import cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.message.Demo01Message;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.messaging.handler.annotation.Payload;
import org.springframework.stereotype.Component;
@Component
public class Demo01Consumer {
private Logger logger = LoggerFactory.getLogger(getClass());
@StreamListener(value = MySink.DEMO01_INPUT, condition = "headers['tag'] == 'yunai'")
public void onMessage(@Payload Demo01Message message) {
logger.info("[onMessage][线程编号:{} 消息内容:{}]", Thread.currentThread().getId(), message);
}
}

View File

@@ -0,0 +1,13 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.listener;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.messaging.SubscribableChannel;
public interface MySink {
String DEMO01_INPUT = "demo01-input";
@Input(DEMO01_INPUT)
SubscribableChannel demo01Input();
}

View File

@@ -0,0 +1,29 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.message;
/**
* 示例 01 的 Message 消息
*/
public class Demo01Message {
/**
* 编号
*/
private Integer id;
public Demo01Message setId(Integer id) {
this.id = id;
return this;
}
public Integer getId() {
return id;
}
@Override
public String toString() {
return "Demo01Message{" +
"id=" + id +
'}';
}
}

View File

@@ -0,0 +1,22 @@
spring:
application:
name: demo-consumer-application
cloud:
# Spring Cloud Stream 配置项,对应 BindingServiceProperties 类
stream:
# Binder 配置项,对应 BinderProperties Map
# binders:
# Binding 配置项,对应 BindingProperties Map
bindings:
demo01-input:
destination: DEMO-TOPIC-01 # 目的地。这里使用 Kafka Topic
content-type: application/json # 内容格式。这里使用 JSON
group: demo01-consumer-group # 消费者分组
# Spring Cloud Stream Kafka 配置项
kafka:
# Kafka Binder 配置项,对应 KafkaBinderConfigurationProperties 类
binder:
brokers: 127.0.0.1:9092 # 指定 Kafka Broker 地址,可以设置多个,以逗号分隔
server:
port: ${random.int[10000,19999]} # 随机端口,方便启动多个消费者

View File

@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>labx-11</artifactId>
<groupId>cn.iocoder.springboot.labs</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>labx-11-sc-stream-kafka-consumer-partitioning</artifactId>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring.boot.version>2.2.4.RELEASE</spring.boot.version>
<spring.cloud.version>Hoxton.SR1</spring.cloud.version>
</properties>
<!--
引入 Spring Boot、Spring Cloud、Spring Cloud Alibaba 三者 BOM 文件,进行依赖版本的管理,防止不兼容。
在 https://dwz.cn/mcLIfNKt 文章中Spring Cloud Alibaba 开发团队推荐了三者的依赖关系
-->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>${spring.boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring.cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- 引入 SpringMVC 相关依赖,并实现对其的自动配置 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- 引入 Spring Cloud Stream Kafka 相关依赖,将 Kafka 作为消息队列,并实现对其的自动配置 -->
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,16 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo;
import cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.listener.MySink;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
@SpringBootApplication
@EnableBinding(MySink.class)
public class ConsumerApplication {
public static void main(String[] args) {
SpringApplication.run(ConsumerApplication.class, args);
}
}

View File

@@ -0,0 +1,19 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.listener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.messaging.Message;
import org.springframework.stereotype.Component;
@Component
public class Demo01Consumer {
private Logger logger = LoggerFactory.getLogger(getClass());
@StreamListener(MySink.DEMO01_INPUT)
public void onMessage(Message<?> message) {
logger.info("[onMessage][线程编号:{} 消息内容:{}]", Thread.currentThread().getId(), message);
}
}

View File

@@ -0,0 +1,13 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.listener;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.messaging.SubscribableChannel;
public interface MySink {
String DEMO01_INPUT = "demo01-input";
@Input(DEMO01_INPUT)
SubscribableChannel demo01Input();
}

View File

@@ -0,0 +1,29 @@
package cn.iocoder.springcloud.labx11.kafkademo.consumerdemo.message;
/**
* 示例 01 的 Message 消息
*/
public class Demo01Message {
/**
* 编号
*/
private Integer id;
public Demo01Message setId(Integer id) {
this.id = id;
return this;
}
public Integer getId() {
return id;
}
@Override
public String toString() {
return "Demo01Message{" +
"id=" + id +
'}';
}
}

View File

@@ -0,0 +1,25 @@
spring:
application:
name: demo-consumer-application
cloud:
# Spring Cloud Stream 配置项,对应 BindingServiceProperties 类
stream:
# Binder 配置项,对应 BinderProperties Map
# binders:
# Binding 配置项,对应 BindingProperties Map
bindings:
demo01-input:
destination: DEMO-TOPIC-01 # 目的地。这里使用 Kafka Topic
content-type: application/json # 内容格式。这里使用 JSON
group: demo01-consumer-group # 消费者分组
# Consumer 配置项,对应 ConsumerProperties 类
consumer:
concurrency: 2 # 每个 Consumer 消费线程数的初始大小,默认为 1
# Spring Cloud Stream Kafka 配置项
kafka:
# Kafka Binder 配置项,对应 KafkaBinderConfigurationProperties 类
binder:
brokers: 127.0.0.1:9092 # 指定 Kafka Broker 地址,可以设置多个,以逗号分隔
server:
port: ${random.int[10000,19999]} # 随机端口,方便启动多个消费者

View File

@@ -34,4 +34,20 @@ public class Demo01Controller {
return mySource.demo01Output().send(springMessage);
}
@GetMapping("/send_tag")
public boolean sendTag() {
for (String tag : new String[]{"yunai", "yutou", "tudou"}) {
// 创建 Message
Demo01Message message = new Demo01Message()
.setId(new Random().nextInt());
// 创建 Spring Message 对象
Message<Demo01Message> springMessage = MessageBuilder.withPayload(message)
.setHeader("tag", tag) // 设置 Tag
.build();
// 发送消息
mySource.demo01Output().send(springMessage);
}
return true;
}
}

View File

@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>labx-11</artifactId>
<groupId>cn.iocoder.springboot.labs</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>labx-11-sc-stream-kafka-producer-partitioning</artifactId>
<properties>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<spring.boot.version>2.2.4.RELEASE</spring.boot.version>
<spring.cloud.version>Hoxton.SR1</spring.cloud.version>
</properties>
<!--
引入 Spring Boot、Spring Cloud、Spring Cloud Alibaba 三者 BOM 文件,进行依赖版本的管理,防止不兼容。
在 https://dwz.cn/mcLIfNKt 文章中Spring Cloud Alibaba 开发团队推荐了三者的依赖关系
-->
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>${spring.boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring.cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- 引入 SpringMVC 相关依赖,并实现对其的自动配置 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- 引入 Spring Cloud Stream Kafka 相关依赖,将 Kafka 作为消息队列,并实现对其的自动配置 -->
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,16 @@
package cn.iocoder.springcloud.labx11.kafkademo.kafkademo;
import cn.iocoder.springcloud.labx11.kafkademo.kafkademo.message.MySource;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
@SpringBootApplication
@EnableBinding(MySource.class)
public class ProducerApplication {
public static void main(String[] args) {
SpringApplication.run(ProducerApplication.class, args);
}
}

View File

@@ -0,0 +1,41 @@
package cn.iocoder.springcloud.labx11.kafkademo.kafkademo.controller;
import cn.iocoder.springcloud.labx11.kafkademo.kafkademo.message.Demo01Message;
import cn.iocoder.springcloud.labx11.kafkademo.kafkademo.message.MySource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.Message;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.Random;
@RestController
@RequestMapping("/demo01")
public class Demo01Controller {
private Logger logger = LoggerFactory.getLogger(getClass());
@Autowired
private MySource mySource;
@GetMapping("/send_orderly")
public boolean sendOrderly() {
// 发送 3 条相同 id 的消息
int id = new Random().nextInt();
for (int i = 0; i < 3; i++) {
// 创建 Message
Demo01Message message = new Demo01Message().setId(id);
// 创建 Spring Message 对象
Message<Demo01Message> springMessage = MessageBuilder.withPayload(message)
.build();
// 发送消息
mySource.demo01Output().send(springMessage);
}
return true;
}
}

View File

@@ -0,0 +1,29 @@
package cn.iocoder.springcloud.labx11.kafkademo.kafkademo.message;
/**
* 示例 01 的 Message 消息
*/
public class Demo01Message {
/**
* 编号
*/
private Integer id;
public Demo01Message setId(Integer id) {
this.id = id;
return this;
}
public Integer getId() {
return id;
}
@Override
public String toString() {
return "Demo01Message{" +
"id=" + id +
'}';
}
}

View File

@@ -0,0 +1,11 @@
package cn.iocoder.springcloud.labx11.kafkademo.kafkademo.message;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
public interface MySource {
@Output("demo01-output")
MessageChannel demo01Output();
}

View File

@@ -0,0 +1,30 @@
spring:
application:
name: demo-producer-application
cloud:
# Spring Cloud Stream 配置项,对应 BindingServiceProperties 类
stream:
# Binder 配置项,对应 BinderProperties Map
# binders:
# Binding 配置项,对应 BindingProperties Map
bindings:
demo01-output:
destination: DEMO-TOPIC-01 # 目的地。这里使用 Kafka Topic
content-type: application/json # 内容格式。这里使用 JSON
# Producer 配置项,对应 ProducerProperties 类
producer:
partition-key-expression: payload['id'] # 分区 key 表达式。该表达式基于 Spring EL从消息中获得分区 key。
# Spring Cloud Stream Kafka 配置项
kafka:
# Kafka Binder 配置项,对应 KafkaBinderConfigurationProperties 类
binder:
brokers: 127.0.0.1:9092 # 指定 Kafka Broker 地址,可以设置多个,以逗号分隔
# Kafka 自定义 Binding 配置项,对应 KafkaBindingProperties Map
bindings:
demo01-output:
# Kafka Producer 配置项,对应 KafkaProducerProperties 类
producer:
sync: true # 是否同步发送消息,默认为 false 异步。
server:
port: 18080

View File

@@ -26,6 +26,13 @@
<!--搭配 <module>labx-11-sc-stream-kafka-producer-demo</module> 作为生产者-->
<module>labx-11-sc-stream-kafka-consumer-concurrency</module>
<module>labx-11-sc-stream-kafka-producer-partitioning</module>
<module>labx-11-sc-stream-kafka-consumer-concurrency</module>
<module>labx-11-sc-stream-kafka-consumer-partitioning</module> <!-- 暂时没用到 -->
<!--搭配 <module>labx-11-sc-stream-kafka-producer-demo</module> 作为生产者-->
<module>labx-11-sc-stream-kafka-consumer-filter</module>
</modules>
</project>