add fluentd and elastic search

.env

@@ -0,0 +1 @@
+ELASTIC_PASSWORD=abc123

.gitignore

@@ -1,5 +1,6 @@
 data/db
 data/logs
+data/elastic*
 *.jbk
 *.jnl
 *.signed

Makefile

@@ -5,7 +5,9 @@ pull:
 	git submodule update --init images/flask/app
 
 init:
+	@mkdir ./data/elasticsearch
 	@mkdir ./data/logs
+	@chmod 777 ./data/elasticsearch
 	@chmod 777 ./data/logs
 	@chmod 777 ./data/zones/
 	@chmod 777 ./config/named/keys/
@@ -55,3 +57,4 @@ rm-db:
 	rm -f data/zones/*.jnl
 	rm -f data/zones/*.signed
 	rm -rf data/logs
+	rm -rf data/elasticsearch

config/fluentd/fluent.conf

@@ -9,11 +9,6 @@
   path /fluentd/logs/db_sql.log
 </match>
 
-<match named.dns>
-  @type file
-  path /fluentd/logs/named_dns.log
-</match>
-
 <match backend.api>
   @type file
   path /fluentd/logs/backend_api.log
@@ -24,6 +19,12 @@
   path /fluentd/logs/backend_thread.log
 </match>
 
-<match **>
-  @type null
+<match named.dns>
+ @type elasticsearch
+  host 172.21.21.7
+  port 9200
+  index_name fluentd.${tag}
+  type_name named
+  user elastic
+  password abc123
 </match>

docker-compose.yaml

@@ -19,7 +19,7 @@ services:
     logging:
       driver: fluentd
       options:
-        fluentd-address: localhost:24224
+        fluentd-address: 127.0.0.1:24224
         tag: db.sql
 
   named:
@@ -38,11 +38,11 @@ services:
     logging:
       driver: fluentd
       options:
-        fluentd-address: localhost:24224
+        fluentd-address: 127.0.0.1:24224
         tag: named.dns
 
   flask_app:
-    build: ./images/flask    
+    build: ./images/flask
     volumes:
       - ./config/named/ddnskey.conf:/etc/ddnskey.conf
     ports:
@@ -57,7 +57,7 @@ services:
     logging:
       driver: fluentd
       options:
-        fluentd-address: localhost:24224
+        fluentd-address: 127.0.0.1:24224
         tag: backend.api
 
   backend_app:
@@ -75,11 +75,11 @@ services:
     logging:
       driver: fluentd
       options:
-        fluentd-address: localhost:24224
+        fluentd-address: 127.0.0.1:24224
         tag: backend.thread
 
   fluentd:
-    image: fluent/fluentd:v1.16.2-debian-1.1
+    build: ./images/fluentd
     container_name: fluentd
     volumes:
       - ./config/fluentd:/fluentd/etc
@@ -90,6 +90,23 @@ services:
     networks:
       docker:
         ipv4_address: 172.21.21.6
+    depends_on:
+      - elasticsearch
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node
+      - cluster.name=elastic-udt-cluster
+      - ELASTIC_USERNAME=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+    volumes:
+      - ./data/elasticsearch:/usr/share/elasticsearch/data
+      - ./data/elasticlog:/usr/share/elasticsearch/logs
+    networks:
+      docker:
+        ipv4_address: 172.21.21.7
 
 networks:
   docker:

images/fluentd/Dockerfile

@@ -0,0 +1,6 @@
+FROM fluent/fluentd:v1.16.2-debian-1.1
+USER root
+RUN gem install elasticsearch -v 7.6
+RUN gem install fluent-plugin-elasticsearch
+RUN gem install fluent-plugin-rewrite-tag-filter
+RUN gem install fluent-plugin-multi-format-parser