OK so I went through the process of setting up ssl as per the instructions here:developers.eventstore.com/server/v5/security.html#setting-up-ssl-for-docker|
This didnt work, the openssl command provided doesn’t actually output any csr file, just a pem.
I then found this documentation:
Which mentions you can’t setup a secure connection on a single node and I need to setup a cluster with certificates.
This did work, based on the below docker compose I am able to get a three cluster node running.
## Inference Pool Docker Compose
#
# Event Store Cluster
# https://developers.eventstore.com/server/v24.6/installation.html#use-docker-compose
# Sets up a 3 node cluster with certficates
#
#
# Nest JS API
# The api of the application. Runs on port 3000
#
# Client (Vite / React)
#
services:
# Setup service for generating certificates
setup:
image: eventstore/es-gencert-cli:1.0.2
entrypoint: bash
user: "1000:1000"
command: >
-c "mkdir -p ./certs && cd /certs && es-gencert-cli create-ca && es-gencert-cli create-node -out ./node1 -ip-addresses 127.0.0.1,172.30.240.11 -dns-names localhost && es-gencert-cli create-node -out ./node2 -ip-addresses 127.0.0.1,172.30.240.12 -dns-names localhost && es-gencert-cli create-node -out ./node3 -ip-addresses 127.0.0.1,172.30.240.13 -dns-names localhost && find . -type f -print0 | xargs -0 chmod 666"
container_name: setup
volumes:
- ./certs:/certs
# Node 1 for Event Store Cluster
node1.eventstore: &template
image: eventstore/eventstore:24.2.0-jammy
container_name: node1.eventstore
env_file:
- vars.env
environment:
- EVENTSTORE_INT_IP=172.30.240.11
- EVENTSTORE_ADVERTISE_HTTP_PORT_TO_CLIENT_AS=2111
- EVENTSTORE_GOSSIP_SEED=172.30.240.12:2113,172.30.240.13:2113
- EVENTSTORE_TRUSTED_ROOT_CERTIFICATES_PATH=/certs/ca
- EVENTSTORE_CERTIFICATE_FILE=/certs/node1/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/certs/node1/node.key
healthcheck:
test: [ "CMD-SHELL", "curl --fail --insecure https://node1.eventstore:2113/health/live || exit 1" ]
interval: 5s
timeout: 5s
retries: 24
ports:
- 2111:2113
volumes:
- ./certs:/certs
- node1-data:/var/lib/eventstore # Data volume for node1
- node1-logs:/var/log/eventstore # Logs volume for node1
depends_on:
- setup
restart: always
networks:
clusternetwork:
ipv4_address: 172.30.240.11
# Node 2 for Event Store Cluster
node2.eventstore:
<<: *template
container_name: node2.eventstore
environment:
- EVENTSTORE_INT_IP=172.30.240.12
- EVENTSTORE_ADVERTISE_HTTP_PORT_TO_CLIENT_AS=2112
- EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.13:2113
- EVENTSTORE_TRUSTED_ROOT_CERTIFICATES_PATH=/certs/ca
- EVENTSTORE_CERTIFICATE_FILE=/certs/node2/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/certs/node2/node.key
healthcheck:
test: [ "CMD-SHELL", "curl --fail --insecure https://node2.eventstore:2113/health/live || exit 1" ]
interval: 5s
timeout: 5s
retries: 24
ports:
- 2112:2113
volumes:
- ./certs:/certs
- node2-data:/var/lib/eventstore # Data volume for node2
- node2-logs:/var/log/eventstore # Logs volume for node2
networks:
clusternetwork:
ipv4_address: 172.30.240.12
# Node 3 for Event Store Cluster
node3.eventstore:
<<: *template
container_name: node3.eventstore
environment:
- EVENTSTORE_INT_IP=172.30.240.13
- EVENTSTORE_ADVERTISE_HTTP_PORT_TO_CLIENT_AS=2113
- EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.12:2113
- EVENTSTORE_TRUSTED_ROOT_CERTIFICATES_PATH=/certs/ca
- EVENTSTORE_CERTIFICATE_FILE=/certs/node3/node.crt
- EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/certs/node3/node.key
healthcheck:
test: [ "CMD-SHELL", "curl --fail --insecure https://node3.eventstore:2113/health/live || exit 1" ]
interval: 5s
timeout: 5s
retries: 24
ports:
- 2113:2113
volumes:
- ./certs:/certs
- node3-data:/var/lib/eventstore # Data volume for node3
- node3-logs:/var/log/eventstore # Logs volume for node3
networks:
clusternetwork:
ipv4_address: 172.30.240.13
# NestJS Application
nestjs-app:
build: .
ports:
- '3000:3000'
depends_on:
- node1.eventstore
- node2.eventstore
- node3.eventstore
environment:
- EVENTSTORE_DB_URL=esdb://admin:[email protected]:2111,node2.eventstore:2112,node3.eventstore:2113?tls=true&tlsVerifyCert=false
# Client Application
client:
build:
context: ./client
ports:
- '5173:5173'
environment:
- VITE_API_URL=http://localhost:5173 # Adjust this based on your API URL
networks:
clusternetwork:
ipv4_address: 172.30.240.15
networks:
clusternetwork:
name: eventstoredb.local
driver: bridge
ipam:
driver: default
config:
- subnet: 172.30.240.0/24
volumes:
eventstore-volume-data:
eventstore-volume-logs:
node1-data: # Volume for node1 data
node1-logs: # Volume for node1 logs
node2-data: # Volume for node2 data
node2-logs: # Volume for node2 logs
node3-data: # Volume for node3 data
node3-logs:
Now I can hit the endpoints via https from post man once the container is started, i can login, i can create users in the UI (Great!) using the default password.
A get request to a node works great outside the container:
https://admin:[email protected]:2112/streams/$all
Now my problem is the connection from my API to the cluster. Below is the connection string im using and believe may be the problem:
EVENTSTORE_DB_URL=esdb://admin:[email protected]:2111,node2.eventstore:2112,node3.eventstore:2113?tls=true&tlsVerifyCert=false
When I call event store from the API the following error is returned:
| Error: Failed to discover after 10 attempts.
nestjs-app-1 | at discoverEndpoint (/app/node_modules/@eventstore/db-client/dist/Client/discovery.js:45:11)
nestjs-app-1 | at runNextTicks (node:internal/process/task_queues:60:5)
nestjs-app-1 | at process.processTimers (node:internal/timers:516:9)
nestjs-app-1 | at async Client.resolveUri (/app/node_modules/@eventstore/db-client/dist/Client/index.js:270:43)
nestjs-app-1 | at async Client.createChannel (/app/node_modules/@eventstore/db-client/dist/Client/index.js:244:25)
nestjs-app-1 | at async Client.createGRPCClient (/app/node_modules/@eventstore/db-client/dist/Client/index.js:200:37)
nestjs-app-1 | at async /app/node_modules/@eventstore/db-client/dist/Client/index.js:145:28
nestjs-app-1 | at async ReadStream.initialize (/app/node_modules/@eventstore/db-client/dist/streams/utils/ReadStream.js:25:18)
