Merge pull request #277 from jembi/TB-412-clickhouse-system-table-ttl

TB-412: Add table config override to set ttl for system log tables
jembi · Apr 5, 2024 · bdb9304 · bdb9304
2 parents f1a2020 + 95dea8f
commit bdb9304
Show file tree

Hide file tree

Showing 6 changed files with 103 additions and 0 deletions.
diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml
@@ -23,6 +23,14 @@ services:
         source: clickhouse_remote_servers.xml
       - target: /etc/clickhouse-server/config.d/use_keeper.xml
         source: clickhouse_use_keeper.xml
+      - target: /etc/clickhouse-server/config.d/metric_log.xml
+        source: clickhouse_metric_log.xml
+      - target: /etc/clickhouse-server/config.d/part_log.xml
+        source: clickhouse_part_log.xml
+      - target: /etc/clickhouse-server/config.d/query_log.xml
+        source: clickhouse_query_log.xml
+      - target: /etc/clickhouse-server/config.d/trace_log.xml
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -49,6 +57,14 @@ services:
         source: clickhouse_remote_servers.xml
       - target: /etc/clickhouse-server/config.d/use_keeper.xml
         source: clickhouse_use_keeper.xml
+      - target: /etc/clickhouse-server/config.d/metric_log.xml
+        source: clickhouse_metric_log.xml
+      - target: /etc/clickhouse-server/config.d/part_log.xml
+        source: clickhouse_part_log.xml
+      - target: /etc/clickhouse-server/config.d/query_log.xml
+        source: clickhouse_query_log.xml
+      - target: /etc/clickhouse-server/config.d/trace_log.xml
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -75,6 +91,14 @@ services:
         source: clickhouse_remote_servers.xml
       - target: /etc/clickhouse-server/config.d/use_keeper.xml
         source: clickhouse_use_keeper.xml
+      - target: /etc/clickhouse-server/config.d/metric_log.xml
+        source: clickhouse_metric_log.xml
+      - target: /etc/clickhouse-server/config.d/part_log.xml
+        source: clickhouse_part_log.xml
+      - target: /etc/clickhouse-server/config.d/query_log.xml
+        source: clickhouse_query_log.xml
+      - target: /etc/clickhouse-server/config.d/trace_log.xml
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -95,6 +119,14 @@ services:
         source: clickhouse_remote_servers.xml
       - target: /etc/clickhouse-server/config.d/use_keeper.xml
         source: clickhouse_use_keeper.xml
+      - target: /etc/clickhouse-server/config.d/metric_log.xml
+        source: clickhouse_metric_log.xml
+      - target: /etc/clickhouse-server/config.d/part_log.xml
+        source: clickhouse_part_log.xml
+      - target: /etc/clickhouse-server/config.d/query_log.xml
+        source: clickhouse_query_log.xml
+      - target: /etc/clickhouse-server/config.d/trace_log.xml
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -156,6 +188,26 @@ configs:
     name: use_keeper.xml-${use_keeper_xml_DIGEST:?err}
     labels: 
       name: clickhouse
+  clickhouse_metric_log.xml:
+    file: ./general_configs/metric_log.xml
+    name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+  clickhouse_part_log.xml:
+    file: ./general_configs/part_log.xml
+    name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+  clickhouse_query_log.xml:
+    file: ./general_configs/query_log.xml
+    name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+  clickhouse_trace_log.xml:
+    file: ./general_configs/trace_log.xml
+    name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
 
 networks:
   public:

diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml
@@ -7,13 +7,44 @@ services:
       noFile: 262144
     volumes:
       - clickhouse-data:/var/lib/clickhouse/
+    configs:
+      - target: /etc/clickhouse-server/config.d/metric_log.xml
+        source: clickhouse_metric_log.xml
+      - target: /etc/clickhouse-server/config.d/part_log.xml
+        source: clickhouse_part_log.xml
+      - target: /etc/clickhouse-server/config.d/query_log.xml
+        source: clickhouse_query_log.xml
+      - target: /etc/clickhouse-server/config.d/trace_log.xml
+        source: clickhouse_trace_log.xml  
     networks:
       public:
       default:
 
 volumes:
   clickhouse-data:
 
+configs:
+  clickhouse_metric_log.xml:
+    file: ./general_configs/metric_log.xml
+    name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+  clickhouse_part_log.xml:
+    file: ./general_configs/part_log.xml
+    name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+  clickhouse_query_log.xml:
+    file: ./general_configs/query_log.xml
+    name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+  clickhouse_trace_log.xml:
+    file: ./general_configs/trace_log.xml
+    name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
+    labels: 
+      name: clickhouse
+
 networks:
   public:
     name: clickhouse_public

diff --git a/analytics-datastore-clickhouse/general_configs/metric_log.xml b/analytics-datastore-clickhouse/general_configs/metric_log.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <metric_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </metric_log>
+</clickhouse>
diff --git a/analytics-datastore-clickhouse/general_configs/part_log.xml b/analytics-datastore-clickhouse/general_configs/part_log.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <part_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </part_log>
+</clickhouse>
diff --git a/analytics-datastore-clickhouse/general_configs/query_log.xml b/analytics-datastore-clickhouse/general_configs/query_log.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <query_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </query_log>
+</clickhouse>
diff --git a/analytics-datastore-clickhouse/general_configs/trace_log.xml b/analytics-datastore-clickhouse/general_configs/trace_log.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <trace_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </trace_log>
+</clickhouse>