commit c7d26917bf5b5f92ef4c9f7ac46aca2c57aeafa2
parent 3cc676cc3c5b4a0735dd0988040ba2d14726052e
Author: Silas Brack <silasbrack@gmail.com>
Date: Sat, 4 Jul 2026 14:07:20 +0200
revert: remove ClickHouse + Vector log pipeline
ClickHouse is overkill for this scale and too memory-hungry for
the 2GB helios server. Reverting to the existing gzipped journalctl
archival — query with DuckDB from gaia instead.
Swap is kept as it's useful regardless.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 0 insertions(+), 192 deletions(-)
diff --git a/hosts/helios.nix b/hosts/helios.nix
@@ -16,7 +16,6 @@
../modules/forgejo.nix
../modules/mkv.nix
../modules/simple-web-app.nix
- ../modules/clickhouse-logs.nix
../modules/daily-tracker.nix
../modules/mailserver.nix
];
@@ -130,9 +129,6 @@
package = simple-web-app.packages.${pkgs.stdenv.hostPlatform.system}.default;
};
- # Log storage (ClickHouse + Vector)
- services.local.clickhouse-logs.enable = true;
-
# Daily Tracker
services.local.daily-tracker = {
enable = false;
diff --git a/modules/clickhouse-logs.nix b/modules/clickhouse-logs.nix
@@ -1,188 +0,0 @@
-{
- config,
- lib,
- pkgs,
- ...
-}:
-
-with lib;
-
-let
- cfg = config.services.local.clickhouse-logs;
-
- initSql = pkgs.writeText "clickhouse-init.sql" ''
- CREATE DATABASE IF NOT EXISTS logs;
-
- CREATE TABLE IF NOT EXISTS logs.simple_web_app (
- timestamp DateTime64(6) DEFAULT now64(6),
- message String,
- priority UInt8 DEFAULT 6,
- unit String,
- syslog_identifier String,
- hostname String
- ) ENGINE = MergeTree()
- ORDER BY timestamp
- TTL toDateTime(timestamp) + INTERVAL ${toString cfg.retentionDays} DAY
- SETTINGS ttl_only_drop_parts = 1;
-
- CREATE TABLE IF NOT EXISTS logs.nginx_access (
- timestamp DateTime64(6) DEFAULT now64(6),
- remote_addr String,
- method String,
- path String,
- status UInt16,
- body_bytes_sent UInt64,
- http_referer String,
- http_user_agent String,
- server_name String
- ) ENGINE = MergeTree()
- ORDER BY timestamp
- TTL toDateTime(timestamp) + INTERVAL ${toString cfg.retentionDays} DAY
- SETTINGS ttl_only_drop_parts = 1;
- '';
-in
-{
- options.services.local.clickhouse-logs = {
- enable = mkEnableOption "ClickHouse log storage with Vector ingestion";
-
- units = mkOption {
- type = types.listOf types.str;
- default = [ "simple-web-app" ];
- description = "Systemd units to collect logs from.";
- };
-
- vpnAddress = mkOption {
- type = types.str;
- default = "10.100.0.7";
- description = "VPN interface address to listen on.";
- };
-
- retentionDays = mkOption {
- type = types.ints.positive;
- default = 90;
- description = "Number of days to retain logs.";
- };
- };
-
- config = mkIf cfg.enable {
- # ClickHouse server — listens on localhost + VPN, memory-constrained
- services.clickhouse = {
- enable = true;
- extraServerConfig = ''
- <clickhouse>
- <listen_host>127.0.0.1</listen_host>
- <listen_host>${cfg.vpnAddress}</listen_host>
- <http_port>8123</http_port>
- <tcp_port>9000</tcp_port>
- <max_server_memory_usage>500000000</max_server_memory_usage>
- </clickhouse>
- '';
- };
-
- # Firewall: allow ClickHouse ports on VPN only
- networking.firewall.interfaces."wg0".allowedTCPPorts = [ 8123 9000 ];
-
- # Init: create database and table after ClickHouse starts
- systemd.services.clickhouse-init = {
- description = "Initialize ClickHouse log tables";
- after = [ "clickhouse.service" ];
- requires = [ "clickhouse.service" ];
- wantedBy = [ "multi-user.target" ];
- path = [ pkgs.clickhouse ];
- script = ''
- for i in $(seq 1 30); do
- clickhouse-client --query "SELECT 1" 2>/dev/null && break
- sleep 1
- done
- clickhouse-client --queries-file ${initSql}
- '';
- serviceConfig = {
- Type = "oneshot";
- RemainAfterExit = true;
- };
- };
-
- # Vector: ship journald logs to ClickHouse
- services.vector = {
- enable = true;
- journaldAccess = true;
- settings = {
- sources.journald = {
- type = "journald";
- include_units = cfg.units;
- };
-
- transforms.parse_journald = {
- type = "remap";
- inputs = [ "journald" ];
- source = ''
- . = {
- "timestamp": format_timestamp!(.timestamp, format: "%Y-%m-%d %H:%M:%S%.6f"),
- "message": to_string(.message) ?? "",
- "priority": to_int(.PRIORITY) ?? 6,
- "unit": to_string(._SYSTEMD_UNIT) ?? "",
- "syslog_identifier": to_string(.SYSLOG_IDENTIFIER) ?? "",
- "hostname": to_string(.host) ?? "",
- }
- '';
- };
-
- sources.nginx_access = {
- type = "file";
- include = [ "/var/log/nginx/access.log" ];
- };
-
- transforms.parse_nginx = {
- type = "remap";
- inputs = [ "nginx_access" ];
- source = ''
- parsed = parse_regex!(.message, r'^(?P<remote_addr>\S+) - \S+ \[(?P<timestamp>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+) \S+" (?P<status>\d+) (?P<body_bytes_sent>\d+) "(?P<http_referer>[^"]*)" "(?P<http_user_agent>[^"]*)"')
- ts = parse_timestamp!(parsed.timestamp, format: "%d/%b/%Y:%H:%M:%S %z")
- . = {
- "timestamp": format_timestamp!(ts, format: "%Y-%m-%d %H:%M:%S%.6f"),
- "remote_addr": parsed.remote_addr,
- "method": parsed.method,
- "path": parsed.path,
- "status": to_int!(parsed.status),
- "body_bytes_sent": to_int!(parsed.body_bytes_sent),
- "http_referer": parsed.http_referer,
- "http_user_agent": parsed.http_user_agent,
- "server_name": "",
- }
- '';
- drop_on_error = true;
- };
-
- sinks.clickhouse_app = {
- type = "clickhouse";
- inputs = [ "parse_journald" ];
- endpoint = "http://127.0.0.1:8123";
- database = "logs";
- table = "simple_web_app";
- skip_unknown_fields = true;
- batch.timeout_secs = 5;
- healthcheck.enabled = true;
- };
-
- sinks.clickhouse_nginx = {
- type = "clickhouse";
- inputs = [ "parse_nginx" ];
- endpoint = "http://127.0.0.1:8123";
- database = "logs";
- table = "nginx_access";
- skip_unknown_fields = true;
- batch.timeout_secs = 5;
- healthcheck.enabled = true;
- };
- };
- };
-
- # Vector must start after ClickHouse is initialized
- # Needs nginx group to read access logs
- systemd.services.vector = {
- after = [ "clickhouse-init.service" ];
- requires = [ "clickhouse-init.service" ];
- serviceConfig.SupplementaryGroups = [ "nginx" ];
- };
- };
-}
