系统环境：centos7
软件版本：jdk1.8、zookeeper3.4.8、hadoop2.8.5
本次实验使用版本 kafka_2.12-3.0.0
一、安装
Kafka官网
将安装包上传至linux服务器上
解压
tar -zxvf kafka_2.12-3.0.0.tgz -C /home/local/
移动目录至kafka
mv kafka_2.12-3.0.0/ kafka
二、部署
配置Kafka环境
vi /etc/profile
添加如下配置
#kafka
export KAFKA_HOME=/home/local/kafka
export PATH=$PATH:${KAFKA_HOME}/bin
修改server.properties文件
vim /home/local/kafka/config/server.properties
修改参数如下：
broker.id=0
listeners=PLAINTEXT://192.168.245.200:9092
log.dirs=/tmp/kafka-logs
zookeeper.connect=192.168.245.200:2181,192.168.245.201:2181,192.168.245.202:2181
参数说明：
broker.id ： 集群内全局唯一标识，每个节点上需要设置不同的值
listeners：这个IP地址也是与本机相关的，每个节点上设置为自己的IP地址
log.dirs ：存放kafka消息的
zookeeper.connect ： 配置的是zookeeper集群地址
分发kafka安装目录
for i in {1..2};do scp -r /home/local/kafka root@slave${i}:/home/local/;done
三、启动
进入kafka安装目录下
./bin/kafka-server-start.sh ./config/server.properties &
kafka相关命令
创建topic
kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
显示所有topic
kafka-topics.sh --list --bootstrap-server localhost:9092
产生消息
kafka-console-producer.sh --broker-list localhost:9092 --topic test
消费消息
kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
删除topic
kafka-topics.sh --delete --bootstrap-server localhost:9092 --topic test
四、flink与kafka结合示例
首先 ，构建maven工程，加入flink与kafka的一些依赖：
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>bigdata-kafka_2.12-3.0.0</artifactId>
<version>1.0-SNAPSHOT</version>
<name>bigdata-kafka_2.12-3.0.0</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<flink-version>1.14.0</flink-version>
<scala.binary.version>2.11.2</scala.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink-version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>${flink-version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>${flink-version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>${flink-version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
第一个，flink生产者示例代码：
package com.example;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import java.io.Serializable;
import java.util.Properties;
public class KafkaProducerExample {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
Properties props = new Properties();
props.setProperty("bootstrap.servers", "192.168.245.200:9092");
DataStream<String> stream = env.addSource(new SimpleStringGenerator());
stream.addSink(new FlinkKafkaProducer<String>("test", new SimpleStringSchema(), props));
env.execute();
}
}
class SimpleStringGenerator implements SourceFunction<String>, Serializable {
private static final long serialVersionUID = 1L;
private volatile boolean isRunning = true;
@Override
public void run(SourceContext<String> ctx) throws Exception {
while (isRunning) {
String str = RandomStringUtils.randomAlphanumeric(5);
ctx.collect(str);
Thread.sleep(1000);
}
}
@Override
public void cancel() {
isRunning = false;
}
}
因为flink是生产者，需要启动一个kafka的消费者终端，然后运行本示例：
启动kafka
bin/kafka-server-start.sh config/server.properties &
启动一个kafka的消费者终端
bin/kafka-console-consumer.sh --bootstrap-server master:9092 --topic test
终端内容
第二个，flink消费者示例代码：
package com.example;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import java.util.Properties;
public class KafkaConsumerApp {
public static void main(String[] args) {
try {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "master:9092");
properties.setProperty("group.id", "flink");
DataStream<String> stream = env.addSource(new FlinkKafkaConsumer<String>("test", new SimpleStringSchema(), properties));
stream.map(new MapFunction<String, Object>() {
@Override
public Object map(String value) throws Exception {
return "flink: " + value;
}
}).print();
env.execute("consumer");
} catch (Exception e) {
e.printStackTrace();
}
}
}
为了测试，我们先开启一个生产者，不断往kafka中发送消息。
kafka-console-producer.sh --broker-list master:9092 --topic test
终端
控制台
打印结果符合预期，flink与kafka结合的示例就演示完成了，主要的还是熟悉flink编程。
