From 44af49cc15b377b6fca33debfcef3c047b834ab6 Mon Sep 17 00:00:00 2001
From: zeus <rootapostolos@swarmlab.io>
Date: Fri, 26 Nov 2021 00:57:19 +0200
Subject: [PATCH] edit fluent

---
 WriteToMongo/fluent/fluent.conf | 110 ++++++++++++++++++++------------
 1 file changed, 68 insertions(+), 42 deletions(-)

diff --git a/WriteToMongo/fluent/fluent.conf b/WriteToMongo/fluent/fluent.conf
index a69235c..bb6687d 100644
--- a/WriteToMongo/fluent/fluent.conf
+++ b/WriteToMongo/fluent/fluent.conf
@@ -9,70 +9,83 @@
 
   path /var/log/*.log
   path_key tailed_path
+  pos_file /tmp/fluentd--1605454018.pos
+  pos_file_compaction_interval 30s
+  refresh_interval 30s
+  skip_refresh_on_startup
 
   tag stats.node
 
-  # parse json
+  enable_stat_watcher false
+  enable_watch_timer false
+  read_from_head true
+  follow_inodes true  # Without this parameter, file rotation causes log duplication
+
   <parse>
-    @type json
-  </parse>
+    @type none
 
-  pos_file /tmp/fluentd--1605454018.pos
+    keep_time_key true
+  </parse>
 </source>
 
-
 # input stats
 <source>
   @type tail
 
   path /var/log-in/*/*
   path_key tailed_path
+  pos_file /tmp/fluentd--1605454014.pos
+  pos_file_compaction_interval 10s
+  refresh_interval 30s
+  skip_refresh_on_startup
 
   tag log.node
 
-  # parse none
+  enable_stat_watcher false
+  enable_watch_timer false
+  read_from_head true
+  follow_inodes true  # Without this parameter, file rotation causes log duplication
   <parse>
     @type none
-  </parse>
 
-  pos_file /tmp/fluentd--1605454014.pos
+    keep_time_key true
+  </parse>
 </source>
 
-
 # output mongo log*
 <match log.*>
   @type copy
-  <store>                              
-          #@type mongo_replset         
+  copy_mode deep
+  <store ignore_error>                              
           @type mongo                  
                                        
-          #database app_swarmlab       
-          database fluentdb            
-          #collection logs             
+	  connection_string mongodb://mongo.poc-datacollector_datacollector-net:27017/fluentdb
+
+          #database fluentdb            
           collection test              
-          #nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
-          host mongo.poc-datacollector_datacollector-net                       
-          port 27017                                                           
-                                                                               
-          #user app_swarmlab                                                   
-          #password app_swarmlab                                               
+          #host mongo.poc-datacollector_datacollector-net                       
+          #port 27017                                                           
                                                                                
-          #replica_set rs0                                                     
           num_retries 60                                                       
           capped                                                               
           capped_size 100m                                                     
-                                                                               
+  <inject>
+    # key name of timestamp
+         time_key time
+    </inject>
                                                                                
           <buffer>                                                             
-            flush_interval 20s                                                 
+            retry_wait 50s
+              flush_mode immediate
+            #flush_interval 10s                                                 
           </buffer>                                                            
   </store>                                                        
 
-  <store>
+  <store ignore_error>
         @type stdout
   </store>
 
-  <store>
+  <store ignore_error>
 	  @type file
 	  path /tmp/mylog
 	  <buffer>
@@ -82,29 +95,42 @@
 	  </buffer>
   </store>
 
-
 </match>
 
-# output mongo stats*
+# output mongo log*
 <match stats.*>
   @type copy
-  <store>
-          @type mongo_replset
-
-          database app_swarmlab
-          collection logs
-          nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
-
-          user swarmlab
-          password swarmlab
+  <store>                              
+          @type mongo                  
+          #copy_mode shallow
+                                       
+          database fluentdb            
+          collection stats              
+          host mongo.poc-datacollector_datacollector-net                       
+          port 27017                                                           
+                                                                               
+          num_retries 60                                                       
+          capped                                                               
+          capped_size 100m                                                     
+                                                                               
+                                                                               
+          <buffer>                                                             
+            flush_interval 30s                                                 
+          </buffer>                                                            
+  </store>                                                        
 
-          replica_set rs0
-          num_retries 60
-          capped
-          capped_size 100m
+  <store>
+        @type stdout
   </store>
+
   <store>
-       @type stdout
+	  @type file
+	  path /tmp/mylog
+	  <buffer>
+	    timekey 1d
+	    timekey_use_utc true
+	    timekey_wait 10s
+	  </buffer>
   </store>
-</match>
 
+</match>