From 95dea8f8b3d0e70dbf9f6614822596ba4e7a39f0 Mon Sep 17 00:00:00 2001 From: Arran Standish Date: Wed, 3 Apr 2024 12:20:27 +0200 Subject: [PATCH] Add table config override to set ttl for system log tables --- .../docker-compose.cluster.yml | 52 +++++++++++++++++++ .../docker-compose.yml | 31 +++++++++++ .../general_configs/metric_log.xml | 5 ++ .../general_configs/part_log.xml | 5 ++ .../general_configs/query_log.xml | 5 ++ .../general_configs/trace_log.xml | 5 ++ 6 files changed, 103 insertions(+) create mode 100644 analytics-datastore-clickhouse/general_configs/metric_log.xml create mode 100644 analytics-datastore-clickhouse/general_configs/part_log.xml create mode 100644 analytics-datastore-clickhouse/general_configs/query_log.xml create mode 100644 analytics-datastore-clickhouse/general_configs/trace_log.xml diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml index 9fd02064..f2c6b61b 100644 --- a/analytics-datastore-clickhouse/docker-compose.cluster.yml +++ b/analytics-datastore-clickhouse/docker-compose.cluster.yml @@ -23,6 +23,14 @@ services: source: clickhouse_remote_servers.xml - target: /etc/clickhouse-server/config.d/use_keeper.xml source: clickhouse_use_keeper.xml + - target: /etc/clickhouse-server/config.d/metric_log.xml + source: clickhouse_metric_log.xml + - target: /etc/clickhouse-server/config.d/part_log.xml + source: clickhouse_part_log.xml + - target: /etc/clickhouse-server/config.d/query_log.xml + source: clickhouse_query_log.xml + - target: /etc/clickhouse-server/config.d/trace_log.xml + source: clickhouse_trace_log.xml networks: public: default: @@ -49,6 +57,14 @@ services: source: clickhouse_remote_servers.xml - target: /etc/clickhouse-server/config.d/use_keeper.xml source: clickhouse_use_keeper.xml + - target: /etc/clickhouse-server/config.d/metric_log.xml + source: clickhouse_metric_log.xml + - target: /etc/clickhouse-server/config.d/part_log.xml + source: clickhouse_part_log.xml + - target: /etc/clickhouse-server/config.d/query_log.xml + source: clickhouse_query_log.xml + - target: /etc/clickhouse-server/config.d/trace_log.xml + source: clickhouse_trace_log.xml networks: public: default: @@ -75,6 +91,14 @@ services: source: clickhouse_remote_servers.xml - target: /etc/clickhouse-server/config.d/use_keeper.xml source: clickhouse_use_keeper.xml + - target: /etc/clickhouse-server/config.d/metric_log.xml + source: clickhouse_metric_log.xml + - target: /etc/clickhouse-server/config.d/part_log.xml + source: clickhouse_part_log.xml + - target: /etc/clickhouse-server/config.d/query_log.xml + source: clickhouse_query_log.xml + - target: /etc/clickhouse-server/config.d/trace_log.xml + source: clickhouse_trace_log.xml networks: public: default: @@ -95,6 +119,14 @@ services: source: clickhouse_remote_servers.xml - target: /etc/clickhouse-server/config.d/use_keeper.xml source: clickhouse_use_keeper.xml + - target: /etc/clickhouse-server/config.d/metric_log.xml + source: clickhouse_metric_log.xml + - target: /etc/clickhouse-server/config.d/part_log.xml + source: clickhouse_part_log.xml + - target: /etc/clickhouse-server/config.d/query_log.xml + source: clickhouse_query_log.xml + - target: /etc/clickhouse-server/config.d/trace_log.xml + source: clickhouse_trace_log.xml networks: public: default: @@ -156,6 +188,26 @@ configs: name: use_keeper.xml-${use_keeper_xml_DIGEST:?err} labels: name: clickhouse + clickhouse_metric_log.xml: + file: ./general_configs/metric_log.xml + name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err} + labels: + name: clickhouse + clickhouse_part_log.xml: + file: ./general_configs/part_log.xml + name: part_log.xml.xml-${part_log_xml_DIGEST:?err} + labels: + name: clickhouse + clickhouse_query_log.xml: + file: ./general_configs/query_log.xml + name: query_log.xml.xml-${query_log_xml_DIGEST:?err} + labels: + name: clickhouse + clickhouse_trace_log.xml: + file: ./general_configs/trace_log.xml + name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err} + labels: + name: clickhouse networks: public: diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml index b5a050cd..ce6363a8 100644 --- a/analytics-datastore-clickhouse/docker-compose.yml +++ b/analytics-datastore-clickhouse/docker-compose.yml @@ -7,6 +7,15 @@ services: noFile: 262144 volumes: - clickhouse-data:/var/lib/clickhouse/ + configs: + - target: /etc/clickhouse-server/config.d/metric_log.xml + source: clickhouse_metric_log.xml + - target: /etc/clickhouse-server/config.d/part_log.xml + source: clickhouse_part_log.xml + - target: /etc/clickhouse-server/config.d/query_log.xml + source: clickhouse_query_log.xml + - target: /etc/clickhouse-server/config.d/trace_log.xml + source: clickhouse_trace_log.xml networks: public: default: @@ -14,6 +23,28 @@ services: volumes: clickhouse-data: +configs: + clickhouse_metric_log.xml: + file: ./general_configs/metric_log.xml + name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err} + labels: + name: clickhouse + clickhouse_part_log.xml: + file: ./general_configs/part_log.xml + name: part_log.xml.xml-${part_log_xml_DIGEST:?err} + labels: + name: clickhouse + clickhouse_query_log.xml: + file: ./general_configs/query_log.xml + name: query_log.xml.xml-${query_log_xml_DIGEST:?err} + labels: + name: clickhouse + clickhouse_trace_log.xml: + file: ./general_configs/trace_log.xml + name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err} + labels: + name: clickhouse + networks: public: name: clickhouse_public diff --git a/analytics-datastore-clickhouse/general_configs/metric_log.xml b/analytics-datastore-clickhouse/general_configs/metric_log.xml new file mode 100644 index 00000000..5b4aa39d --- /dev/null +++ b/analytics-datastore-clickhouse/general_configs/metric_log.xml @@ -0,0 +1,5 @@ + + + event_date + INTERVAL 30 DAY DELETE + + diff --git a/analytics-datastore-clickhouse/general_configs/part_log.xml b/analytics-datastore-clickhouse/general_configs/part_log.xml new file mode 100644 index 00000000..28e68d57 --- /dev/null +++ b/analytics-datastore-clickhouse/general_configs/part_log.xml @@ -0,0 +1,5 @@ + + + event_date + INTERVAL 30 DAY DELETE + + diff --git a/analytics-datastore-clickhouse/general_configs/query_log.xml b/analytics-datastore-clickhouse/general_configs/query_log.xml new file mode 100644 index 00000000..2b67083f --- /dev/null +++ b/analytics-datastore-clickhouse/general_configs/query_log.xml @@ -0,0 +1,5 @@ + + + event_date + INTERVAL 30 DAY DELETE + + diff --git a/analytics-datastore-clickhouse/general_configs/trace_log.xml b/analytics-datastore-clickhouse/general_configs/trace_log.xml new file mode 100644 index 00000000..de08e2fe --- /dev/null +++ b/analytics-datastore-clickhouse/general_configs/trace_log.xml @@ -0,0 +1,5 @@ + + + event_date + INTERVAL 30 DAY DELETE + +