edit fluent

WriteToMongo/fluent/fluent.conf

@@ -9,70 +9,83 @@
 
   path /var/log/*.log
   path_key tailed_path
+  pos_file /tmp/fluentd--1605454018.pos
+  pos_file_compaction_interval 30s
+  refresh_interval 30s
+  skip_refresh_on_startup
 
   tag stats.node
 
-  # parse json
+  enable_stat_watcher false
+  enable_watch_timer false
+  read_from_head true
+  follow_inodes true  # Without this parameter, file rotation causes log duplication
+
   <parse>
-    @type json
-  </parse>
+    @type none
 
-  pos_file /tmp/fluentd--1605454018.pos
+    keep_time_key true
+  </parse>
 </source>
 
-
 # input stats
 <source>
   @type tail
 
   path /var/log-in/*/*
   path_key tailed_path
+  pos_file /tmp/fluentd--1605454014.pos
+  pos_file_compaction_interval 10s
+  refresh_interval 30s
+  skip_refresh_on_startup
 
   tag log.node
 
-  # parse none
+  enable_stat_watcher false
+  enable_watch_timer false
+  read_from_head true
+  follow_inodes true  # Without this parameter, file rotation causes log duplication
   <parse>
     @type none
-  </parse>
 
-  pos_file /tmp/fluentd--1605454014.pos
+    keep_time_key true
+  </parse>
 </source>
 
-
 # output mongo log*
 <match log.*>
   @type copy
-  <store>                              
-          #@type mongo_replset         
+  copy_mode deep
+  <store ignore_error>                              
           @type mongo                  
                                        
-          #database app_swarmlab       
-          database fluentdb            
-          #collection logs             
-          collection test              
-          #nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
-          host mongo.poc-datacollector_datacollector-net                       
-          port 27017                                                           
+	  connection_string mongodb://mongo.poc-datacollector_datacollector-net:27017/fluentdb
 
-          #user app_swarmlab                                                   
-          #password app_swarmlab                                               
+          #database fluentdb            
+          collection test              
+          #host mongo.poc-datacollector_datacollector-net                       
+          #port 27017                                                           
                                                                                
-          #replica_set rs0                                                     
           num_retries 60                                                       
           capped                                                               
           capped_size 100m                                                     
-                                                                               
+  <inject>
+    # key name of timestamp
+         time_key time
+    </inject>
                                                                                
           <buffer>                                                             
-            flush_interval 20s                                                 
+            retry_wait 50s
+              flush_mode immediate
+            #flush_interval 10s                                                 
           </buffer>                                                            
   </store>                                                        
 
-  <store>
+  <store ignore_error>
         @type stdout
   </store>
 
-  <store>
+  <store ignore_error>
 	  @type file
 	  path /tmp/mylog
 	  <buffer>
@@ -82,29 +95,42 @@
 	  </buffer>
   </store>
 
-
 </match>
 
-# output mongo stats*
+# output mongo log*
 <match stats.*>
   @type copy
   <store>                              
-          @type mongo_replset
-
-          database app_swarmlab
-          collection logs
-          nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
+          @type mongo                  
+          #copy_mode shallow
                                        
-          user swarmlab
-          password swarmlab
+          database fluentdb            
+          collection stats              
+          host mongo.poc-datacollector_datacollector-net                       
+          port 27017                                                           
                                                                                
-          replica_set rs0
           num_retries 60                                                       
           capped                                                               
           capped_size 100m                                                     
+                                                                               
+                                                                               
+          <buffer>                                                             
+            flush_interval 30s                                                 
+          </buffer>                                                            
   </store>                                                        
+
   <store>
         @type stdout
   </store>
-</match>
 
+  <store>
+	  @type file
+	  path /tmp/mylog
+	  <buffer>
+	    timekey 1d
+	    timekey_use_utc true
+	    timekey_wait 10s
+	  </buffer>
+  </store>
+
+</match>