From 3d0af194cc329bae6545ef1209559254f4a5b5b7 Mon Sep 17 00:00:00 2001 From: zeus Date: Fri, 2 Apr 2021 19:41:06 +0300 Subject: [PATCH] add mongo example fluentd --- .../fluentd/files/fluent-config-update.conf | 104 +++++++++++++++ .../courses/fluentd/fluentd-config-update.yml | 119 ++++++++++++++++++ .../fluentd/fluentd-config-update.yml.sh | 15 +++ .../courses/fluentd/fluentd-test-mongo.yml | 44 +++++++ .../courses/fluentd/fluentd-test-mongo.yml.sh | 15 +++ 5 files changed, 297 insertions(+) create mode 100755 install/usr/share/swarmlab.io/sec/project/courses/fluentd/files/fluent-config-update.conf create mode 100755 install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml create mode 100755 install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml.sh create mode 100755 install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml create mode 100755 install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml.sh diff --git a/install/usr/share/swarmlab.io/sec/project/courses/fluentd/files/fluent-config-update.conf b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/files/fluent-config-update.conf new file mode 100755 index 0000000..bb43ecc --- /dev/null +++ b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/files/fluent-config-update.conf @@ -0,0 +1,104 @@ +# config + + @type stdout + + +# input log + + @type tail + + path /var/log/*.log + path_key tailed_path + + tag stats.node + + # parse json + + @type json + + + pos_file /tmp/fluentd--1605454018.pos + + + +# input stats + + @type tail + + path /var/log-in/*/* + path_key tailed_path + + tag log.node + + # parse none + + @type none + + + pos_file /tmp/fluentd--1605454014.pos + + + +# output mongo log* + + @type copy + + @type mongo_replset + + database app_swarmlab + collection logs + nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017 + + user app_swarmlab + password app_swarmlab + + replica_set rs0 + num_retries 60 + capped + capped_size 100m + + + + flush_interval 20s + + + + @type stdout + + + + @type file + path /tmp/mylog + + timekey 1d + timekey_use_utc true + timekey_wait 10s + + + + + + +# output mongo stats* + + @type copy + + @type mongo_replset + + database app_swarmlab + collection logs + nodes swarmlabmongo1:27017,swarmlabmongo2:27017,swarmlabmongo1:27017 + + user swarmlab + password swarmlab + + replica_set rs0 + num_retries 60 + capped + capped_size 100m + + + @type stdout + + + diff --git a/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml new file mode 100755 index 0000000..de20705 --- /dev/null +++ b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml @@ -0,0 +1,119 @@ +--- +- hosts: service + remote_user: docker + gather_facts: no + vars: + user: "docker" + + tasks: + +# -------------------------------------------------------------------------------------- +# --------------------- copy conf to fluentd +# -------------------------------------------------------------------------------------- + + # ------------------------ + # cp fluentd.conf + # ------------------------- + - name: cp fluentd.conf + become: true + copy: + src: "./files/fluent-config-update.conf" + dest: /fluentd/etc/fluent.conf + owner: docker + group: docker + mode: 0755 + +# -------------------------------------------------------------------------------------- +# --------------------- kill and save ps tp tmp +# -------------------------------------------------------------------------------------- + + # ------------------------ + # start fluentd + # ------------------------- + - name: find fluentd + shell: ps efw -opid -Cfluentd | grep -v grep | grep -E '[0-9]' + #shell: "ps efw -opid,cmd -Cfluentd | pgrep -o fluentd" + register: fluentdps +# when: fluentdps is defined + +# - fail: msg="this play requires fluentdps" + #when: fluentdps is not defined + + # ------------------------ + # start fluentd + # # ------------------------- + - name: kill -9 fluentd + become: true + ignore_errors: yes + shell: "kill -9 {{ item }}" + with_items: "{{ fluentdps.stdout_lines }}" + when: fluentdps.stdout_lines is defined + +# - fail: msg="this play requires fluentdps" +# when: fluentdps is not defined + + - name: ls fluentdps + debug: var=fluentdps.stdout + + # # ------------------------ + # # save variable > /tmp + # # ------------------------- + # - name: echo kill > tmp + # shell: "echo {{ fluentdps.stdout }} > /tmp/123" + # when: not fluentdps + # + +# -------------------------------------------------------------------------------------- +# --------------------- start and save ps tp tmp +# -------------------------------------------------------------------------------------- + + # ------------------------ + # start fluentd + # ------------------------- + - name: start fluentd background + shell: nohup /home/docker/.gem/ruby/2.5.0/bin/fluentd -c /fluentd/etc/fluent.conf -vv /dev/null 2>&1 & + + # ------------------------ + # start fluentd + # ------------------------- + - name: find1 fluentd + shell: ps efw -opid,cmd -Cfluentd | pgrep -o fluentd + register: fluentdps1 + + - name: ls fluentdps1 + debug: var=fluentdps1.stdout + + # ------------------------ + # save variable > /tmp1 + # ------------------------- + - name: echo > tmp1 + shell: "echo {{ fluentdps1.stdout }} > /tmp/12345" + + # ------------------------ + # example4net tcpdump example + # ------------------------- +# - name: google.com +# become: yes +# become_user: "{{ user }}" +# command: curl http://www.google.com +# ignore_errors: yes +# register: configwww +# +# - name: ls configwww +# debug: var=configwww.stdout_lines +# +# - name: ls -al /var/lab/playground/playground-readmongo/ +# become: yes +# become_user: "{{ user }}" +# #command: ls -al /var/lab/playground/playground-readmongo +# command: netstat -antlupe +# ignore_errors: yes +# register: config +# +# - name: ls config +# debug: var=config.stdout_lines +# +# - name: Refresh connection +# meta: clear_host_errors +# + diff --git a/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml.sh b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml.sh new file mode 100755 index 0000000..4b8bf73 --- /dev/null +++ b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-config-update.yml.sh @@ -0,0 +1,15 @@ +#!/bin/sh + + +ip4=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) +ip6=$(/sbin/ip -o -6 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + +echo "[service]" > /project/courses/fluentd/inventory.yml +/project/bin/swarmlab-nmap >> /project/courses/fluentd/inventory.yml + + +# include master or not +echo $ip4 >> /project/courses/fluentd/inventory.yml + + +ansible-playbook -u docker -i inventory.yml fluentd-config-update.yml -f 5 --ask-pass --ask-become-pass diff --git a/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml new file mode 100755 index 0000000..7d04fcb --- /dev/null +++ b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml @@ -0,0 +1,44 @@ +--- +- hosts: service + remote_user: docker + gather_facts: no + vars: + user: "docker" + + tasks: + +# -------------------------------------------------------------------------------------- +# --------------------- create test dir +# -------------------------------------------------------------------------------------- + + # ------------------------ + # test dir + # ------------------------- + - name: make dir for test + become: true + file: + path: "/var/log-in/test" + state: directory + owner: docker + group: docker + mode: '0755' + +# -------------------------------------------------------------------------------------- +# --------------------- kill and save ps tp tmp +# -------------------------------------------------------------------------------------- + + - name: find fluentd + #shell: df -h >> /var/log-in/test/test + shell: df -h + #shell: "ps efw -opid,cmd -Cfluentd | pgrep -o fluentd" + register: fluentddate + + - name: ls fluentddate + debug: var=fluentddate.stdout_lines + + - name: write to /var/log-in/test/test2 + shell: "echo {{ item }} >> /var/log-in/test/test2" + with_items: "{{ fluentddate.stdout_lines }}" + + + diff --git a/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml.sh b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml.sh new file mode 100755 index 0000000..323902e --- /dev/null +++ b/install/usr/share/swarmlab.io/sec/project/courses/fluentd/fluentd-test-mongo.yml.sh @@ -0,0 +1,15 @@ +#!/bin/sh + + +ip4=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) +ip6=$(/sbin/ip -o -6 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + +echo "[service]" > /project/courses/fluentd/inventory.yml +/project/bin/swarmlab-nmap >> /project/courses/fluentd/inventory.yml + + +# include master or not +echo $ip4 >> /project/courses/fluentd/inventory.yml + + +ansible-playbook -u docker -i inventory.yml fluentd-test-mongo.yml -f 5 --ask-pass --ask-become-pass