From 7e5f2d5a6bbc83053376a57f0dafeadf75b8a5f2 Mon Sep 17 00:00:00 2001
From: zeus <rootapostolos@swarmlab.io>
Date: Sun, 21 Nov 2021 19:21:35 +0200
Subject: [PATCH] add fluentd

---
 WriteToMongo/fluent/fluent.conf | 126 +++++++++++++++++++++++---------
 docker-compose.yml              |   1 +
 2 files changed, 93 insertions(+), 34 deletions(-)

diff --git a/WriteToMongo/fluent/fluent.conf b/WriteToMongo/fluent/fluent.conf
index 54b5592..4a06fe1 100644
--- a/WriteToMongo/fluent/fluent.conf
+++ b/WriteToMongo/fluent/fluent.conf
@@ -1,50 +1,108 @@
+# config
+<match debug.*>
+  @type stdout
+</match>
+
+# input log
 <source>
-  @type forward
-  port 24224
-  bind 0.0.0.0
+  @type tail
+
+  path /var/log/*.log
+  path_key tailed_path
+
+  tag stats.node
+
+  # parse json
+  <parse>
+    @type json
+  </parse>
+
+  pos_file /tmp/fluentd--1605454018.pos
 </source>
 
-<filter mongo.redis>
-  @type record_transformer
-  <record>
-    gen_host "#{Socket.gethostname}"
-  </record>
-  
-</filter>
 
-<filter mongo.*>
-  @type record_transformer
-  <record>
-  tag ${tag}
-  </record>
-</filter>
+# input stats
+<source>
+  @type tail
+
+  path /var/log-in/*/*
+  path_key tailed_path
+
+  tag log.node
+
+  # parse none
+  <parse>
+    @type none
+  </parse>
+
+  pos_file /tmp/fluentd--1605454014.pos
+</source>
 
 
-<match mongo.*>
+# output mongo log*
+<match log.*>
   @type copy
   <store>
-    @type stdout
+          #@type mongo_replset
+          @type mongo
+
+          #database app_swarmlab
+          database fluentdb
+          #collection logs
+          collection test
+          #nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
+          nodes mongo.poc-datacollector_datacollector-net:27017
+
+          #user app_swarmlab
+          #password app_swarmlab
+
+          #replica_set rs0
+          num_retries 60
+          capped
+          capped_size 100m
+
+
+          <buffer>
+            flush_interval 20s
+          </buffer>
   </store>
   <store>
-  @type mongo
+        @type stdout
+  </store>
 
-  host mongo
-  port 27017
+  <store>
+	  @type file
+	  path /tmp/mylog
+	  <buffer>
+	    timekey 1d
+	    timekey_use_utc true
+	    timekey_wait 10s
+	  </buffer>
+  </store>
 
-  database fluentdb
-  collection test
 
-  capped
-  capped_size 1024m
+</match>
 
-  <buffer>
-    flush_interval 10s
-  </buffer>
-  
-  <format>
-    @type json
-  </format>
+# output mongo stats*
+<match stats.*>
+  @type copy
+  <store>
+          @type mongo_replset
+
+          database app_swarmlab
+          collection logs
+          nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
+
+          user swarmlab
+          password swarmlab
+
+          replica_set rs0
+          num_retries 60
+          capped
+          capped_size 100m
+  </store>
+  <store>
+       @type stdout
   </store>
-    # interval
-  
 </match>
+
diff --git a/docker-compose.yml b/docker-compose.yml
index 7f9fca1..208c34a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -66,6 +66,7 @@ services:
     #  - "3001:3001"
     volumes:
        - ./dummyservice/swarmlab-app/src/run/:/home/node/dummyservice/dummyservice/swarmlab-app/src/run/
+       - ./WriteToMongo/fluent/fluent.conf:/fluentd/etc/fluent.conf
     #links:
     #  - mongo
     networks: