diff --git a/README.md b/README.md index 6e2f331..488fc1c 100644 --- a/README.md +++ b/README.md @@ -280,8 +280,11 @@ A collection of delicious docker recipes. - [x] browserless/chrome - [x] certbot - [x] codercom/code-server -- [x] confluentinc/cp-kafka-mqtt -- [x] confluentinc/cp-kafka-rest +- [x] confluentinc + - [x] cp-kafka-mqtt + - [x] cp-kafka-rest + - [x] ksqldb-cli + - [x] ksqldb-server - [x] streamsets/datacollector - [x] daskdev - [x] dask diff --git a/kafka-rest/docker-compose.yml b/kafka-rest/docker-compose.yml index c6a70a8..ee2a444 100644 --- a/kafka-rest/docker-compose.yml +++ b/kafka-rest/docker-compose.yml @@ -1,16 +1,29 @@ -kafka-rest: - image: confluentinc/cp-kafka-rest - ports: - - "8082:8082" - environment: - - KAFKA_REST_HOST_NAME=kafka-rest - - KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 - - KAFKA_REST_BOOTSTRAP_SERVERS=kafka1:9092,kafka2:9092,kafka3:9092 - extra_hosts: - - zookeeper1:10.0.0.21 - - zookeeper2:10.0.0.22 - - zookeeper3:10.0.0.23 - - kafka1:10.0.0.21 - - kafka2:10.0.0.22 - - kafka3:10.0.0.23 - restart: unless-stopped +version: "3.7" + +services: + + schema-registry: + image: confluentinc/cp-schema-registry:5.4.0 + container_name: schema-registry + hostname: schema-registry + environment: + - SCHEMA_REGISTRY_HOST_NAME=schema-registry + - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 + extra_hosts: + - zookeeper:10.0.0.21 + restart: unless-stopped + + kafka-rest: + image: confluentinc/cp-kafka-rest:5.4.0 + container_name: kafka-rest + hostname: kafka-rest + ports: + - "8082:8082" + environment: + - KAFKA_REST_HOST_NAME=kafka-rest + - KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_REST_BOOTSTRAP_SERVERS=kafka:9092 + extra_hosts: + - zookeeper:10.0.0.21 + - kafka:10.0.0.21 + restart: unless-stopped diff --git a/ksqldb/README.md b/ksqldb/README.md new file mode 100644 index 0000000..0e5b606 --- /dev/null +++ b/ksqldb/README.md @@ -0,0 +1,28 @@ +ksqldb +====== + +[ksqlDB][1] is an event streaming database. + +## Up and Running + +```bash +$ docker-compose up -d +$ curl http://127.0.0.1:8088/info +$ docker-compose exec ksqldb-cli ksql http://ksql-server:8088 +>>> CREATE TABLE products (product_name VARCHAR, cost DOUBLE) + WITH (kafka_topic='products', partitions=3, value_format='json'); +>>> CREATE STREAM products_changelog AS + SELECT * FROM products EMIT CHANGES; +>>> SHOW TOPICS; +>>> SHOW TABLES; +>>> SHOW STREAMS; +``` + +## Docs + +- ksql: https://docs.confluent.io/current/ksql/docs/index.html +- ksql-server: https://docs.confluent.io/current/ksql/docs/installation/server-config/index.html +- ksql-cli: https://docs.confluent.io/current/ksql/docs/installation/cli-config.html +- connector: https://docs.ksqldb.io/en/latest/concepts/connectors/ + +[1]: https://ksqldb.io/ diff --git a/ksqldb/data/connect.properties b/ksqldb/data/connect.properties new file mode 100644 index 0000000..9d29bc4 --- /dev/null +++ b/ksqldb/data/connect.properties @@ -0,0 +1,12 @@ +bootstrap.servers=kafka:9092 +plugin.path=/data/plugins +group.id=ksql-connect-cluster +key.converter=org.apache.kafka.connect.json.JsonConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +value.converter.schemas.enable=false +config.storage.topic=ksql-connect-configs +offset.storage.topic=ksql-connect-offsets +status.storage.topic=ksql-connect-statuses +config.storage.replication.factor=1 +offset.storage.replication.factor=1 +status.storage.replication.factor=1 diff --git a/ksqldb/data/plugins/README.md b/ksqldb/data/plugins/README.md new file mode 100644 index 0000000..dadbd1b --- /dev/null +++ b/ksqldb/data/plugins/README.md @@ -0,0 +1,3 @@ +- https://www.confluent.io/hub/confluentinc/kafka-connect-jdbc +- https://www.confluent.io/hub/mongodb/kafka-connect-mongodb +- https://www.confluent.io/hub/neo4j/kafka-connect-neo4j diff --git a/ksqldb/docker-compose.yml b/ksqldb/docker-compose.yml new file mode 100644 index 0000000..6e08373 --- /dev/null +++ b/ksqldb/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.7" + +services: + + ksqldb-server: + image: confluentinc/ksqldb-server:0.6.0 + ports: + - "8088:8088" + environment: + - KSQL_LISTENERS=http://0.0.0.0:8088 + - KSQL_BOOTSTRAP_SERVERS=kafka:9092 + - KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE=true + - KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE=true + - KSQL_KSQL_CONNECT_WORKER_CONFIG=/data/connect.properties + # KSQL_KSQL_CONNECT_URL=kafka-connect:8083 + # KSQL_KSQL_SCHEMA_REGISTRY_URL=http://schema-registry:8081 + volumes: + - ./data:/data + restart: unless-stopped + + ksqldb-cli: + image: confluentinc/ksqldb-cli:0.6.0 + entrypoint: /bin/sh + tty: true + depends_on: + - ksqldb-server + + kafkacat: + image: confluentinc/cp-kafkacat:5.4.0 + entrypoint: /bin/sh + tty: true + depends_on: + - kafka