update ClickHouse to cloud prod version

.github/ISSUE_TEMPLATE/best-place-for-self-hosting-issues-is-the-community-forum.md

@@ -1,10 +0,0 @@
----
-name: Best place for self-hosting issues is the community forum
-about: Please do not submit issues here
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Please do not submit issues here. Best place for self-hosting issues, questions and ideas is [the community forum here on GitHub](https://github.com/plausible/analytics/discussions/categories/self-hosted-support). Thanks!

.gitignore

@@ -0,0 +1,8 @@
+*
+!compose.yml
+!clickhouse/logs.xml
+!clickhouse/ipv4-only.xml
+!clickhouse/low-resources.xml
+!README.md
+!LICENSE
+!.gitignore

.prettierignore

@@ -1 +0,0 @@
-*

clickhouse/clickhouse-config.xml

@@ -1,16 +0,0 @@
-<clickhouse>
-    <logger>
-        <level>warning</level>
-        <console>true</console>
-    </logger>
-
-    <!-- Stop all the unnecessary logging -->
-    <query_thread_log remove="remove"/>
-    <query_log remove="remove"/>
-    <text_log remove="remove"/>
-    <trace_log remove="remove"/>
-    <metric_log remove="remove"/>
-    <asynchronous_metric_log remove="remove"/>
-    <session_log remove="remove"/>
-    <part_log remove="remove"/>
-</clickhouse>

clickhouse/clickhouse-user-config.xml

@@ -1,8 +0,0 @@
-<clickhouse>
-    <profiles>
-        <default>
-            <log_queries>0</log_queries>
-            <log_query_threads>0</log_query_threads>
-        </default>
-    </profiles>
-</clickhouse>

clickhouse/ipv4-only.xml

@@ -0,0 +1,3 @@
+<clickhouse>
+    <listen_host>0.0.0.0</listen_host>
+</clickhouse>

clickhouse/logs.xml

@@ -0,0 +1,28 @@
+<clickhouse>
+    <logger>
+        <level>warning</level>
+        <console>true</console>
+    </logger>
+
+    <query_log replace="1">
+        <database>system</database>
+        <table>query_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+        <engine>
+            ENGINE = MergeTree
+            PARTITION BY event_date
+            ORDER BY (event_time)
+            TTL event_date + interval 30 day
+            SETTINGS ttl_only_drop_parts=1
+        </engine>
+    </query_log>
+
+    <!-- Stops unnecessary logging -->
+    <metric_log remove="remove" />
+    <asynchronous_metric_log remove="remove" />
+    <query_thread_log remove="remove" />
+    <text_log remove="remove" />
+    <trace_log remove="remove" />
+    <session_log remove="remove" />
+    <part_log remove="remove" />
+</clickhouse>

clickhouse/low-resources.xml

@@ -0,0 +1,23 @@
+<!-- https://clickhouse.com/docs/en/operations/tips#using-less-than-16gb-of-ram -->
+<clickhouse>
+    <!--
+    https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#mark_cache_size -->
+    <mark_cache_size>524288000</mark_cache_size>
+
+    <profile>
+        <detault>
+            <!-- https://clickhouse.com/docs/en/operations/settings/settings#max_threads -->
+            <max_threads>1</max_threads>
+            <!-- https://clickhouse.com/docs/en/operations/settings/settings#max_block_size -->
+            <max_block_size>8192</max_block_size>
+            <!-- https://clickhouse.com/docs/en/operations/settings/settings#max_download_threads -->
+            <max_download_threads>1</max_download_threads>
+            <!--
+            https://clickhouse.com/docs/en/operations/settings/settings#input_format_parallel_parsing -->
+            <input_format_parallel_parsing>0</input_format_parallel_parsing>
+            <!--
+            https://clickhouse.com/docs/en/operations/settings/settings#output_format_parallel_formatting -->
+            <output_format_parallel_formatting>0</output_format_parallel_formatting>
+        </detault>
+    </profile>
+</clickhouse>

compose.yml

@@ -0,0 +1,93 @@
+services:
+  plausible_db:
+    image: postgres:16-alpine
+    restart: always
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      start_period: 1m
+
+  plausible_events_db:
+    image: clickhouse/clickhouse-server:24.9.3.128-alpine
+    restart: always
+    volumes:
+      - event-data:/var/lib/clickhouse
+      - event-logs:/var/log/clickhouse-server
+      - ./clickhouse/logs.xml:/etc/clickhouse-server/config.d/logs.xml:ro
+      # This makes ClickHouse bind to IPv4 only, since Docker doesn't enable IPv6 in bridge networks by default.
+      # Fixes "Listen [::]:9000 failed: Address family for hostname not supported" warnings.
+      - ./clickhouse/ipv4-only.xml:/etc/clickhouse-server/config.d/ipv4-only.xml:ro
+      # This makes ClickHouse consume less resources, which is useful for small setups.
+      # https://clickhouse.com/docs/en/operations/tips#using-less-than-16gb-of-ram
+      - ./clickhouse/low-resources.xml:/etc/clickhouse-server/config.d/low-resources.xml:ro
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 -O - http://127.0.0.1:8123/ping || exit 1"]
+      start_period: 1m
+
+  plausible:
+    image: ghcr.io/plausible/community-edition:v2.1.4
+    restart: always
+    command: sh -c "/entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
+    depends_on:
+      plausible_db:
+        condition: service_healthy
+      plausible_events_db:
+        condition: service_healthy
+    volumes:
+      - plausible-data:/var/lib/plausible
+    ulimits:
+      nofile:
+        soft: 65535
+        hard: 65535
+    environment:
+      - TMPDIR=/var/lib/plausible/tmp
+      # required: https://github.com/plausible/community-edition/wiki/configuration#required
+      - BASE_URL=${BASE_URL}
+      - SECRET_KEY_BASE=${SECRET_KEY_BASE}
+      # optional: https://github.com/plausible/community-edition/wiki/configuration#optional
+      # registration: https://github.com/plausible/community-edition/wiki/configuration#registration
+      - TOTP_VAULT_KEY
+      - DISABLE_REGISTRATION
+      - ENABLE_EMAIL_VERIFICATION
+      # web: https://github.com/plausible/community-edition/wiki/configuration#web
+      - HTTP_PORT
+      - HTTPS_PORT
+      # databases: https://github.com/plausible/community-edition/wiki/configuration#database
+      - DATABASE_URL
+      - CLICKHOUSE_DATABASE_URL
+      # Google: https://github.com/plausible/community-edition/wiki/configuration#google
+      - GOOGLE_CLIENT_ID
+      - GOOGLE_CLIENT_SECRET
+      # geolocation: https://github.com/plausible/community-edition/wiki/configuration#ip-geolocation
+      - IP_GEOLOCATION_DB
+      - GEONAMES_SOURCE_FILE
+      - MAXMIND_LICENSE_KEY
+      - MAXMIND_EDITION
+      # email: https://github.com/plausible/community-edition/wiki/configuration#email
+      - MAILER_ADAPTER
+      - MAILER_EMAIL
+      - MAILER_NAME
+      - SMTP_HOST_ADDR
+      - SMTP_HOST_PORT
+      - SMTP_USER_NAME
+      - SMTP_USER_PWD
+      - SMTP_HOST_SSL_ENABLED
+      - POSTMARK_API_KEY
+      - MAILGUN_API_KEY
+      - MAILGUN_DOMAIN
+      - MAILGUN_BASE_URI
+      - MANDRILL_API_KEY
+      - SENDGRID_API_KEY
+
+volumes:
+  db-data:
+  event-data:
+  event-logs:
+  plausible-data:

docker-compose.yml

@@ -1,43 +0,0 @@
-services:
-  plausible_db:
-    # Plausible v2.1.0 was tested against PostgreSQL versions 15 and 16
-    # https://github.com/plausible/analytics/blob/v2.1.0/.github/workflows/elixir.yml#L21-L32
-    image: postgres:16-alpine
-    restart: always
-    volumes:
-      - db-data:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD=postgres
-
-  plausible_events_db:
-    image: clickhouse/clickhouse-server:24.3.3.102-alpine
-    restart: always
-    volumes:
-      - event-data:/var/lib/clickhouse
-      - event-logs:/var/log/clickhouse-server
-      - ./clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro
-      - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro
-    ulimits:
-      nofile:
-        soft: 262144
-        hard: 262144
-
-  plausible:
-    image: ghcr.io/plausible/community-edition:v2.1.0
-    restart: always
-    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
-    depends_on:
-      - plausible_db
-      - plausible_events_db
-    ports:
-      - 127.0.0.1:8000:8000
-    env_file:
-      - plausible-conf.env
-
-volumes:
-  db-data:
-    driver: local
-  event-data:
-    driver: local
-  event-logs:
-    driver: local

plausible-conf.env

@@ -1,3 +0,0 @@
-BASE_URL=replace-me
-SECRET_KEY_BASE=replace-me
-TOTP_VAULT_KEY=replace-me

reverse-proxy/README.md

@@ -1,47 +0,0 @@
-This directory contains pre-made configurations for various reverse proxies. Which flavor you should choose depends on your setup.
-
-## No existing reverse proxy
-
-If you aren't running an existing reverse proxy, then you can use the [`caddy-gen`](https://github.com/wemake-services/caddy-gen) based docker-compose file. Update it to include the domain name you use for your server, then combine it with the existing docker-compose files:
-
-```shell
-$ docker-compose -f docker-compose.yml -f reverse-proxy/docker-compose.caddy-gen.yml up
-```
-
-## Existing reverse proxy
-
-If you are already running a reverse proxy, then the above will not work as it will clash with the existing port bindings. You should instead use one of the available configuration files:
-
-### NGINX
-
-If you already have NGINX running as a system service, use the configuration file in the `nginx` directory.
-
-Edit the file `reverse-proxy/nginx/plausible` to contain the domain name you use for your server, then copy it into NGINX's configuration folder. Enable it by creating a symlink in NGINX's enabled sites folder. Finally use Certbot to create a TLS certificate for your site:
-
-```shell
-$ sudo cp reverse-proxy/nginx/plausible /etc/nginx/sites-available
-$ sudo ln -s /etc/nginx/sites-available/plausible /etc/nginx/sites-enabled/plausible
-$ sudo certbot --nginx
-```
-
-### Traefik 2
-
-If you already have a Traefik container running on Docker, use the docker-compose file in the `traefik` directory. Note that it assumes that your Traefik container is set up to support certificate generation.
-
-Edit the file `reverse-proxy/traefik/docker-compose.traefik.yml` to contain the domain name you use for your server, then combine it with the existing docker-compose files:
-
-```shell
-$ docker-compose -f docker-compose.yml -f reverse-proxy/traefik/docker-compose.traefik.yml up
-```
-
-### Apache2
-Install the necessary Apache modules and restart Apache. Edit the file `reverse-proxy/apache2/plausible.conf` to contain the domain name you use for your server, then copy it into Apache's configuration folder. Enable it by creating a symlink in Apache's enabled sites folder with `a2ensite` command. Finally use Certbot to create a TLS certificate for your site:
-
-```shell
-$ sudo a2enmod proxy proxy_http proxy_ajp remoteip headers proxy_wstunnel
-$ sudo systemctl restart apache2
-$ sudo cp reverse-proxy/apache2/plausible.conf /etc/apache2/sites-available/
-$ sudo a2ensite plausible.conf
-$ sudo systemctl restart apache2
-$ sudo certbot --apache
-```

reverse-proxy/apache2/plausible.conf

@@ -1,18 +0,0 @@
-<VirtualHost *:80>
-
- ServerAdmin [email protected]
- ServerName example.com
-
- ProxyPreserveHost On
- ProxyPass / http://localhost:8000/
- ProxyPassReverse / http://localhost:8000/
-
- SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
- LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
- LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" forwarded
-
- ErrorLog ${APACHE_LOG_DIR}/error.log
- CustomLog ${APACHE_LOG_DIR}/access.log combined env=!forwarded
- CustomLog ${APACHE_LOG_DIR}/access.log forwarded env=forwarded
-    
-</VirtualHost>

reverse-proxy/docker-compose.caddy-gen.yml

@@ -1,24 +0,0 @@
-version: "3.3"
-services:
-  caddy-gen:
-    container_name: caddy-gen
-    image: "wemakeservices/caddy-gen:latest"
-    restart: always
-    volumes:
-      - /var/run/docker.sock:/tmp/docker.sock:ro
-      - caddy-certificates:/data/caddy
-    ports:
-      - "80:80"
-      - "443:443"
-    depends_on:
-      - plausible
-
-  plausible:
-    labels:
-      virtual.host: "example.com" # change to your domain name
-      virtual.port: "8000"
-      virtual.tls-email: "[email protected]" # change to your email
-        
-volumes:
-    caddy-certificates:
-        driver: local

reverse-proxy/nginx/plausible

@@ -1,19 +0,0 @@
-server {
-	# replace example.com with your domain name
-	server_name example.com;
-	
-	listen 80;
-	listen [::]:80;
-
-	location / {
-		proxy_pass http://127.0.0.1:8000;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-
-		location = /live/websocket {
-			proxy_pass http://127.0.0.1:8000;
-			proxy_http_version 1.1;
-			proxy_set_header Upgrade $http_upgrade;
-			proxy_set_header Connection "Upgrade";
-		}
-	}
-}

reverse-proxy/traefik/docker-compose.traefik.yml

@@ -1,8 +0,0 @@
-version: "3.3"
-services:
-  plausible:
-    labels:
-      traefik.enable: "true"
-      traefik.http.routers.plausible.rule: "Host(`example.com`)" # change to your domain name
-      traefik.http.routers.plausible.entrypoints: "websecure"
-      traefik.http.services.plausible.loadbalancer.server.port: "8000"

upgrade/postgres.md

@@ -1,159 +0,0 @@
-Guide to upgrading PostgreSQL version `>= 12` to version `14` using `pg_dump` and `psql`. Based on [Upgrade a PostgreSQL database with docker.](https://hollo.me/devops/upgrade-postgresql-database-with-docker.html)
-
-**Note:** following this guide you'd need to stop some containers and it would make your plausible instance temporarily unavailable.
-
----
-
-### Plan
-
-1. dump contents of the old version PostgreSQL to a file
-1. copy the dump to the host
-1. replace old version PostgreSQL with new version PostgreSQL
-1. copy and load the dump into new version PostgreSQL
-
----
-
-### Steps
-
-1. Stop `plausible` to avoid writing to old `plausible_db`
-
-```console
-> docker compose stop plausible
-[+] Running 2/2
- ⠿ Container hosting-plausible-1     Stopped           6.5s
-```
-
-2. Dump old `plausible_db` contents to a backup file
-
-```console
-> docker compose exec plausible_db sh -c "pg_dump -U postgres plausible_db > plausible_db.bak"
-```
-
-3. Copy the backup to the host
-
-```console
-> docker compose cp plausible_db:plausible_db.bak plausible_db.bak
-```
-
-4. (Optional) verify backup went OK
-
-```console
-> head plausible_db.bak
---
--- PostgreSQL database dump
---
-
--- Dumped from database version 12.12 (Debian 12.12-1.pgdg110+1)
--- Dumped by pg_dump version 12.12 (Debian 12.12-1.pgdg110+1)
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-```
-
-5. Edit `docker-compose.yml` to use new PostgreSQL version, here we update from `v12` to `v14`, alpine flavour.
-
-```diff
-  plausible_db:
--   image: postgres:12
-+   image: postgres:14-alpine
-    restart: always
-    volumes:
-      - db-data:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_PASSWORD=postgres
-```
-
-6. Ensure relevant containers are stopped
-
-```console
-> docker compose stop plausible plausible_db
-[+] Running 2/2
- ⠿ Container hosting-plausible-1     Stopped           0.0s
- ⠿ Container hosting-plausible_db-1  Stopped           0.2s
-```
-
-7. Remove old `plausible_db` container to be able to nuke its volume in the next step
-
-```console
-> docker compose rm plausible_db
-? Going to remove hosting-plausible_db-1 Yes
-[+] Running 1/0
- ⠿ Container hosting-plausible_db-1  Removed           0.0s
-```
-
-8. Remove old `plausible_db` volume, mine is named `hosting_db-data`
-
-```console
-> docker volume ls
-DRIVER    VOLUME NAME
-<...snip...>
-local     hosting_db-data
-local     hosting_event-data
-<...snip...>
-
-> docker volume rm hosting_db-data
-hosting_db-data
-```
-
-9. Start new version `plausible_db` container
-
-```console
-> docker compose up plausible_db -d
-[+] Running 9/9
- ⠿ plausible_db Pulled                                 9.3s
-   ⠿ 9b18e9b68314 Already exists                       0.0s
-   ⠿ 75aada9edfc5 Pull complete                        1.2s
-   ⠿ 820773693750 Pull complete                        1.2s
-   ⠿ 8812bb04ef2e Pull complete                        5.2s
-   ⠿ 2ccec0f7805c Pull complete                        5.2s
-   ⠿ 833f9b98598e Pull complete                        5.3s
-   ⠿ 1eb578dc04e6 Pull complete                        5.4s
-   ⠿ c873bf6204df Pull complete                        5.4s
-[+] Running 2/2
- ⠿ Volume "hosting_db-data"          Created           0.0s
- ⠿ Container hosting-plausible_db-1  Started           0.5s
-```
-
-10. Create new DB and load data into it
-
-```console
-> docker compose exec plausible_db createdb -U postgres plausible_db
-> docker compose cp plausible_db.bak plausible_db:plausible_db.bak
-
-> docker compose exec plausible_db sh -c "psql -U postgres -d plausible_db < plausible_db.bak"
-SET
-SET
-SET
-SET
-SET
- set_config
-------------
-
-(1 row)
-
-SET
-SET
-SET
-SET
-CREATE EXTENSION
-<...snip...>
-```
-
-11. Start all other containers
-
-```console
-> docker compose up -d
-[+] Running 4/4
- ⠿ Container hosting-plausible_events_db-1  Running           0.0s
- ⠿ Container hosting-mail-1                 Running           0.0s
- ⠿ Container hosting-plausible_db-1         Started           0.5s
- ⠿ Container hosting-plausible-1            Started           0.5s
-```
-
-12. (Optional) Remove backups from the container and the host
-
-```console
-> rm plausible_db.bak
-> docker compose exec plausible_db rm plausible_db.bak
-```