add mongo example fluentd

install/usr/share/swarmlab.io/sec/project/courses/fluentd/files/fluent-config-update.conf

@@ -0,0 +1,104 @@
+# config
+<match debug.*>
+  @type stdout
+</match>
+
+# input log
+<source>
+  @type tail
+
+  path /var/log/*.log
+  path_key tailed_path
+
+  tag stats.node
+
+  # parse json
+  <parse>
+    @type json
+  </parse>
+
+  pos_file /tmp/fluentd--1605454018.pos
+</source>
+
+
+# input stats
+<source>
+  @type tail
+
+  path /var/log-in/*/*
+  path_key tailed_path
+
+  tag log.node
+
+  # parse none
+  <parse>
+    @type none
+  </parse>
+
+  pos_file /tmp/fluentd--1605454014.pos
+</source>
+
+
+# output mongo log*
+<match log.*>
+  @type copy
+  <store>
+          @type mongo_replset
+
+          database app_swarmlab
+          collection logs
+          nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
+
+          user app_swarmlab
+          password app_swarmlab
+
+          replica_set rs0
+          num_retries 60
+          capped
+          capped_size 100m
+
+
+          <buffer>
+            flush_interval 20s
+          </buffer>
+  </store>
+  <store>
+        @type stdout
+  </store>
+
+  <store>
+	  @type file
+	  path /tmp/mylog
+	  <buffer>
+	    timekey 1d
+	    timekey_use_utc true
+	    timekey_wait 10s
+	  </buffer>
+  </store>
+
+
+</match>
+
+# output mongo stats*
+<match stats.*>
+  @type copy
+  <store>
+          @type mongo_replset
+
+          database app_swarmlab
+          collection logs
+          nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017
+
+          user swarmlab
+          password swarmlab
+
+          replica_set rs0
+          num_retries 60
+          capped
+          capped_size 100m
+  </store>
+  <store>
+       @type stdout
+  </store>
+</match>
+

install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml

@@ -0,0 +1,119 @@
+---
+- hosts: service
+  remote_user: docker
+  gather_facts: no
+  vars:
+    user: "docker"
+ 
+  tasks:
+
+# --------------------------------------------------------------------------------------
+# ---------------------  copy conf to fluentd
+# --------------------------------------------------------------------------------------
+
+     # ------------------------
+     # cp fluentd.conf
+     # -------------------------
+    - name: cp fluentd.conf
+      become: true
+      copy:
+        src: "./files/fluent-config-update.conf"
+        dest: /fluentd/etc/fluent.conf
+        owner: docker
+        group: docker
+        mode: 0755
+
+# --------------------------------------------------------------------------------------
+# ---------------------  kill and save ps tp tmp
+# --------------------------------------------------------------------------------------
+
+     # ------------------------
+     # start fluentd
+     # -------------------------
+    - name: find fluentd 
+      shell: ps efw -opid -Cfluentd |  grep -v grep | grep -E '[0-9]'
+      #shell: "ps efw -opid,cmd -Cfluentd | pgrep -o fluentd"
+      register: fluentdps
+#      when: fluentdps is defined
+
+#    - fail: msg="this play requires fluentdps"
+      #when: fluentdps is not defined
+
+     # ------------------------
+     # start fluentd
+     #     # -------------------------
+    - name: kill -9 fluentd
+      become: true
+      ignore_errors: yes
+      shell: "kill -9 {{ item }}" 
+      with_items: "{{ fluentdps.stdout_lines }}"
+      when: fluentdps.stdout_lines is defined
+
+#    - fail: msg="this play requires fluentdps"
+#      when: fluentdps is not defined
+     
+    - name: ls  fluentdps
+      debug: var=fluentdps.stdout
+
+      #     # ------------------------
+      #     # save variable > /tmp
+      #     # -------------------------
+      #    - name: echo kill > tmp
+      #      shell: "echo {{ fluentdps.stdout }} > /tmp/123"
+      #      when: not fluentdps
+      #
+
+# --------------------------------------------------------------------------------------
+# ---------------------  start and save ps tp tmp
+# --------------------------------------------------------------------------------------
+
+     # ------------------------
+     # start fluentd
+     # -------------------------
+    - name: start fluentd background
+      shell: nohup /home/docker/.gem/ruby/2.5.0/bin/fluentd -c /fluentd/etc/fluent.conf -vv </dev/null >/dev/null 2>&1 &
+
+     # ------------------------
+     # start fluentd
+     # -------------------------
+    - name: find1 fluentd 
+      shell: ps efw -opid,cmd -Cfluentd | pgrep -o fluentd
+      register: fluentdps1
+
+    - name: ls  fluentdps1
+      debug: var=fluentdps1.stdout
+
+     # ------------------------
+     # save variable > /tmp1
+     # -------------------------
+    - name: echo > tmp1
+      shell: "echo {{ fluentdps1.stdout }} > /tmp/12345"
+
+     # ------------------------
+     # example4net tcpdump example
+     # -------------------------
+#    - name: google.com
+#      become: yes
+#      become_user: "{{ user }}"
+#      command:  curl http://www.google.com
+#      ignore_errors: yes
+#      register: configwww
+#
+#    - name: ls  configwww
+#      debug: var=configwww.stdout_lines
+#
+#    - name: ls -al /var/lab/playground/playground-readmongo/
+#      become: yes
+#      become_user: "{{ user }}"
+#      #command:  ls -al /var/lab/playground/playground-readmongo
+#      command:  netstat -antlupe
+#      ignore_errors: yes
+#      register: config
+#
+#    - name: ls  config
+#      debug: var=config.stdout_lines
+#
+#    - name: Refresh connection
+#      meta: clear_host_errors
+#
+ 

install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+
+ip4=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
+ip6=$(/sbin/ip -o -6 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
+
+echo "[service]" > /project/courses/fluentd/inventory.yml
+/project/bin/swarmlab-nmap >> /project/courses/fluentd/inventory.yml
+
+
+# include master or not
+echo $ip4 >> /project/courses/fluentd/inventory.yml
+
+
+ansible-playbook -u docker -i inventory.yml fluentd-config-update.yml  -f 5  --ask-pass --ask-become-pass

install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml

@@ -0,0 +1,44 @@
+---
+- hosts: service
+  remote_user: docker
+  gather_facts: no
+  vars:
+    user: "docker"
+ 
+  tasks:
+
+# --------------------------------------------------------------------------------------
+# ---------------------  create test dir
+# --------------------------------------------------------------------------------------
+
+     # ------------------------
+     # test dir
+     # -------------------------
+    - name: make dir for test
+      become: true
+      file:
+        path: "/var/log-in/test"
+        state: directory
+        owner: docker
+        group: docker
+        mode: '0755'
+
+# --------------------------------------------------------------------------------------
+# ---------------------  kill and save ps tp tmp
+# --------------------------------------------------------------------------------------
+
+    - name: find fluentd 
+      #shell:  df -h >> /var/log-in/test/test
+      shell:  df -h
+      #shell: "ps efw -opid,cmd -Cfluentd | pgrep -o fluentd"
+      register: fluentddate
+     
+    - name: ls  fluentddate
+      debug: var=fluentddate.stdout_lines
+
+    - name: write to  /var/log-in/test/test2
+      shell:  "echo {{ item }} >> /var/log-in/test/test2"
+      with_items: "{{ fluentddate.stdout_lines }}"
+
+
+ 

install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+
+ip4=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
+ip6=$(/sbin/ip -o -6 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
+
+echo "[service]" > /project/courses/fluentd/inventory.yml
+/project/bin/swarmlab-nmap >> /project/courses/fluentd/inventory.yml
+
+
+# include master or not
+echo $ip4 >> /project/courses/fluentd/inventory.yml
+
+
+ansible-playbook -u docker -i inventory.yml fluentd-test-mongo.yml  -f 5  --ask-pass --ask-become-pass