kafka部署
kafka
https://kafka.apache.org/
到kafka官网下载kafka binary包，解压到本地
运行脚本
在主目录中配置run.bat脚本，调用bin里面的bat脚本启动kafka
start bin\windows\zookeeper-server-start.bat config\zookeeper.properties
start bin\windows\kafka-server-start.bat config\server.properties
关闭使用
start bin\windows\zookeeper-server-stop.bat
start bin\windows\kafka-server-stop.bat
命令行
- 创建主题（3分区、2副本）
bin\windows\kafka-topics.bat --create --topic test-topic --partitions 3 --replication-factor 2 --bootstrap-server localhost:9092 - 查看所有主题列表
bin\windows\kafka-topics.bat --list --bootstrap-server localhost:9092
图形化展示
Kafka Eagle：https://www.kafka-eagle.org/
修改config\system-config.properties，zookeeper的地址
cluster1.zk.list=localhost:2181
配置mysql，数据库ke，并创建表如下
CREATE TABLE IF NOT EXISTS `ke_alarm_crontab` (`id` BIGINT ( 20 ) NOT NULL,`type` VARCHAR ( 64 ) NOT NULL,`crontab` VARCHAR ( 32 ) DEFAULT NULL,`is_enable` VARCHAR ( 2 ) DEFAULT 'Y',`created` VARCHAR ( 32 ) DEFAULT NULL,`modify` VARCHAR ( 32 ) DEFAULT NULL,PRIMARY KEY ( `id`, `type` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_metrics` (`cluster` VARCHAR ( 64 ) DEFAULT NULL,`broker` text DEFAULT NULL,`type` VARCHAR ( 32 ) DEFAULT NULL,`key` VARCHAR ( 64 ) DEFAULT NULL,`value` VARCHAR ( 128 ) DEFAULT NULL,`timespan` BIGINT ( 20 ) DEFAULT NULL,`tm` VARCHAR ( 16 ) DEFAULT NULL ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_p_role` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`name` VARCHAR ( 64 ) CHARACTER SET utf8 NOT NULL COMMENT 'role name',`seq` TINYINT ( 4 ) NOT NULL COMMENT 'rank',`description` VARCHAR ( 128 ) CHARACTER SET utf8 NOT NULL COMMENT 'role describe',PRIMARY KEY ( `id` ) ) ENGINE = INNODB AUTO_INCREMENT = 4 DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_alarm_clusters` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`type` VARCHAR ( 32 ) DEFAULT NULL,`cluster` VARCHAR ( 64 ) DEFAULT NULL,`server` text DEFAULT NULL,`alarm_group` VARCHAR ( 128 ) DEFAULT NULL,`alarm_times` INT ( 11 ),`alarm_max_times` INT ( 11 ),`alarm_level` VARCHAR ( 4 ),`is_normal` VARCHAR ( 2 ) DEFAULT 'Y',`is_enable` VARCHAR ( 2 ) DEFAULT 'Y',`created` VARCHAR ( 32 ) DEFAULT NULL,`modify` VARCHAR ( 32 ) DEFAULT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_alarm_consumer` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`cluster` VARCHAR ( 64 ) DEFAULT NULL,`group` VARCHAR ( 128 ) DEFAULT NULL,`topic` VARCHAR ( 128 ) DEFAULT NULL,`lag` BIGINT ( 20 ) DEFAULT NULL,`alarm_group` VARCHAR ( 128 ) DEFAULT NULL,`alarm_times` INT ( 11 ),`alarm_max_times` INT ( 11 ),`alarm_level` VARCHAR ( 4 ),`is_normal` VARCHAR ( 2 ) DEFAULT 'Y',`is_enable` VARCHAR ( 2 ) DEFAULT 'Y',`created` VARCHAR ( 32 ) DEFAULT NULL,`modify` VARCHAR ( 32 ) DEFAULT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_connect_config` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`cluster` VARCHAR ( 64 ),`connect_uri` VARCHAR ( 128 ),`version` VARCHAR ( 32 ),`alive` VARCHAR ( 16 ),`created` VARCHAR ( 32 ) DEFAULT NULL,`modify` VARCHAR ( 32 ) DEFAULT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_resources` (`resource_id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`name` VARCHAR ( 255 ) CHARACTER SET utf8 NOT NULL COMMENT 'resource name',`url` VARCHAR ( 255 ) NOT NULL,`parent_id` INT ( 11 ) NOT NULL,PRIMARY KEY ( `resource_id` ) ) ENGINE = INNODB AUTO_INCREMENT = 17 DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_metrics_offline` (`cluster` VARCHAR ( 64 ) NOT NULL,`key` VARCHAR ( 128 ) NOT NULL,`one` VARCHAR ( 128 ) DEFAULT NULL,`mean` VARCHAR ( 128 ) DEFAULT NULL,`five` VARCHAR ( 128 ) DEFAULT NULL,`fifteen` VARCHAR ( 128 ) DEFAULT NULL,PRIMARY KEY ( `cluster`, `key` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_logsize` (`cluster` VARCHAR ( 64 ) DEFAULT NULL,`topic` VARCHAR ( 64 ) DEFAULT NULL,`logsize` BIGINT ( 20 ) DEFAULT NULL,`diffval` BIGINT ( 20 ) DEFAULT NULL,`timespan` BIGINT ( 20 ) DEFAULT NULL,`tm` VARCHAR ( 16 ) DEFAULT NULL ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_consumer_group_summary` (`cluster` VARCHAR ( 64 ) NOT NULL,`group` VARCHAR ( 128 ) NOT NULL,`topic_number` VARCHAR ( 128 ) NOT NULL,`coordinator` VARCHAR ( 128 ) DEFAULT NULL,`active_topic` INT ( 11 ) DEFAULT NULL,`active_thread_total` INT ( 11 ) DEFAULT NULL,PRIMARY KEY ( `cluster`, `group` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_consumer_bscreen` (`cluster` VARCHAR ( 64 ) DEFAULT NULL,`group` VARCHAR ( 128 ) DEFAULT NULL,`topic` VARCHAR ( 64 ) DEFAULT NULL,`logsize` BIGINT ( 20 ) DEFAULT NULL,`difflogsize` BIGINT ( 20 ) DEFAULT NULL,`offsets` BIGINT ( 20 ) DEFAULT NULL,`diffoffsets` BIGINT ( 20 ) DEFAULT NULL,`lag` BIGINT ( 20 ) DEFAULT NULL,`timespan` BIGINT ( 20 ) DEFAULT NULL,`tm` VARCHAR ( 16 ) DEFAULT NULL ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_users` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`rtxno` INT ( 11 ) NOT NULL,`username` VARCHAR ( 64 ) NOT NULL,`password` VARCHAR ( 128 ) NOT NULL,`email` VARCHAR ( 64 ) NOT NULL,`realname` VARCHAR ( 128 ) NOT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB AUTO_INCREMENT = 2 DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_sql_history` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`cluster` VARCHAR ( 64 ) DEFAULT NULL,`username` VARCHAR ( 64 ) DEFAULT NULL,`host` VARCHAR ( 128 ) DEFAULT NULL,`ksql` text DEFAULT NULL,`status` VARCHAR ( 16 ) DEFAULT NULL,`spend_time` BIGINT ( 20 ) DEFAULT NULL,`created` VARCHAR ( 32 ) DEFAULT NULL,`tm` VARCHAR ( 16 ) DEFAULT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_consumer_group` (`cluster` VARCHAR ( 64 ) NOT NULL,`group` VARCHAR ( 128 ) NOT NULL,`topic` VARCHAR ( 128 ) NOT NULL,`status` INT ( 11 ) DEFAULT NULL,PRIMARY KEY ( `cluster`, `group`, `topic` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_alarm_config` (`cluster` VARCHAR ( 64 ) NOT NULL,`alarm_group` VARCHAR ( 128 ) NOT NULL,`alarm_type` VARCHAR ( 16 ) DEFAULT NULL,`alarm_url` text DEFAULT NULL,`http_method` VARCHAR ( 16 ) DEFAULT NULL,`alarm_address` text DEFAULT NULL,`created` VARCHAR ( 32 ) DEFAULT NULL,`modify` VARCHAR ( 32 ) DEFAULT NULL,PRIMARY KEY ( `cluster`, `alarm_group` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_user_role` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`user_id` INT ( 11 ) NOT NULL,`role_id` TINYINT ( 4 ) NOT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB AUTO_INCREMENT = 2 DEFAULT CHARSET = utf8mb4;CREATE TABLE IF NOT EXISTS `ke_role_resource` (`id` BIGINT ( 20 ) NOT NULL AUTO_INCREMENT,`role_id` INT ( 11 ) NOT NULL,`resource_id` INT ( 11 ) NOT NULL,PRIMARY KEY ( `id` ) ) ENGINE = INNODB AUTO_INCREMENT = 19 DEFAULT CHARSET = utf8mb4;ALTER TABLE `ke_consumer_bscreen` ADD INDEX `idx_timespan` ( `timespan` );
ALTER TABLE `ke_logsize` ADD INDEX `idx_timespan` ( `timespan` );
ALTER TABLE `ke_logsize` ADD INDEX `idx_tm_topic` ( `tm`, `topic` );
ALTER TABLE `ke_logsize` ADD INDEX `idx_tm_cluster_diffval` ( `tm`, `cluster`, `diffval` );
ALTER TABLE `ke_consumer_bscreen` ADD INDEX `idx_tm_cluster_diffoffsets` ( `tm`, `cluster`, `diffoffsets` );
INSERT INTO `ke_users`
VALUES( '1', '1000', 'admin', '123456', 'admin@email.com', 'Administrator' );;
CREATE TABLE IF NOT EXISTS `ke_topic_rank` (`cluster` VARCHAR ( 64 ) NOT NULL,`topic` VARCHAR ( 64 ) NOT NULL,`tkey` VARCHAR ( 64 ) NOT NULL,`tvalue` BIGINT ( 20 ) DEFAULT NULL,PRIMARY KEY ( `cluster`, `topic`, `tkey` ) ) ENGINE = INNODB DEFAULT CHARSET = utf8mb4;;
INSERT INTO `ke_user_role`
VALUES( '1', '1', '1' );;INSERT INTO `ke_p_role`
VALUES( '1', 'Administrator', '1', 'Have all permissions' ),( '2', 'Devs', '2', 'Own add or delete' ),( '3', 'Tourist', '3', 'Only viewer' );INSERT INTO `ke_role_resource`
VALUES( '1', '1', '1' ),( '2', '1', '2' ),( '3', '1', '3' ),( '4', '1', '4' ),( '5', '1', '5' ),( '6', '1', '7' ),( '7', '1', '8' ),( '8', '1', '10' ),( '9', '1', '11' ),( '10', '1', '13' ),( '11', '2', '7' ),( '12', '2', '8' ),( '13', '2', '13' ),( '14', '2', '10' ),( '15', '2', '11' ),( '16', '1', '14' ),( '17', '1', '15' ),( '18', '1', '16' ),( '19', '1', '18' ),( '20', '1', '19' ),( '21', '1', '20' ),( '22', '1', '21' ),( '23', '1', '22' ),( '24', '1', '23' ),( '25', '1', '24' );
INSERT INTO `ke_resources`VALUES( '1', 'System', '/system', '-1' ),( '2', 'User', '/system/user', '1' ),( '3', 'Role', '/system/role', '1' ),( '4', 'Resource', '/system/resource', '1' ),( '5', 'Notice', '/system/notice', '1' ),( '6', 'Topic', '/topic', '-1' ),( '7', 'Message', '/topic/message', '6' ),( '8', 'Create', '/topic/create', '6' ),( '9', 'Alarm', '/alarm', '-1' ),( '10', 'Add', '/alarm/add', '9' ),( '11', 'Modify', '/alarm/modify', '9' ),( '12', 'Cluster', '/cluster', '-1' ),( '13', 'ZkCli', '/cluster/zkcli', '12' ),( '14', 'UserDelete', '/system/user/delete', '1' ),( '15', 'UserModify', '/system/user/modify', '1' ),( '16', 'Mock', '/topic/mock', '6' ),( '18', 'Create', '/alarm/create', '9' ),( '19', 'History', '/alarm/history', '9' ),( '20', 'Manager', '/topic/manager', '6' ),( '21', 'PasswdReset', '/system/user/reset', '1' ),( '22', 'Config', '/alarm/config', '9' ),( '23', 'List', '/alarm/list', '9' ),( '24', 'Hub', '/topic/hub', '6' );
配置环境变量后运行脚本
set KE_HOME=D:\Tools\kafka_2.13-3.9.1\kafka-eagle-bin-3.0.1\efak-web-3.0.1
set Path=%KE_HOME%\bin
start kafka-eagle-bin-3.0.1\efak-web-3.0.1\bin\ke.bat
http://localhost:8048，默认账号admin，密码123456
异常
- java.sql.SQLException: The server time zone value ‘?й???’ is unrecognized or represents more than one time zone.
config\system-config.properties，添加serverTimezone=UTC
efak.url=jdbc:mysql://127.0.0.1:3306/ke?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&serverTimezone=UTC