From d9e2903726eaaadbbca6b20192f73f454676f794 Mon Sep 17 00:00:00 2001 From: thaleia-kavalierou Date: Sun, 13 Jun 2021 13:14:16 +0000 Subject: [PATCH] Upload files to 'fluentd' --- fluentd/fluentd-config-update.yml | 119 ++++++++++++++++ fluentd/fluentd-config-update.yml.sh | 15 ++ fluentd/fluentd-test-mongo.yml | 54 ++++++++ fluentd/fluentd.yml | 198 +++++++++++++++++++++++++++ fluentd/fluentd.yml.sh | 25 ++++ 5 files changed, 411 insertions(+) create mode 100644 fluentd/fluentd-config-update.yml create mode 100644 fluentd/fluentd-config-update.yml.sh create mode 100644 fluentd/fluentd-test-mongo.yml create mode 100644 fluentd/fluentd.yml create mode 100644 fluentd/fluentd.yml.sh diff --git a/fluentd/fluentd-config-update.yml b/fluentd/fluentd-config-update.yml new file mode 100644 index 0000000..de20705 --- /dev/null +++ b/fluentd/fluentd-config-update.yml @@ -0,0 +1,119 @@ +--- +- hosts: service + remote_user: docker + gather_facts: no + vars: + user: "docker" + + tasks: + +# -------------------------------------------------------------------------------------- +# --------------------- copy conf to fluentd +# -------------------------------------------------------------------------------------- + + # ------------------------ + # cp fluentd.conf + # ------------------------- + - name: cp fluentd.conf + become: true + copy: + src: "./files/fluent-config-update.conf" + dest: /fluentd/etc/fluent.conf + owner: docker + group: docker + mode: 0755 + +# -------------------------------------------------------------------------------------- +# --------------------- kill and save ps tp tmp +# -------------------------------------------------------------------------------------- + + # ------------------------ + # start fluentd + # ------------------------- + - name: find fluentd + shell: ps efw -opid -Cfluentd | grep -v grep | grep -E '[0-9]' + #shell: "ps efw -opid,cmd -Cfluentd | pgrep -o fluentd" + register: fluentdps +# when: fluentdps is defined + +# - fail: msg="this play requires fluentdps" + #when: fluentdps is not defined + + # ------------------------ + # start fluentd + # # ------------------------- + - name: kill -9 fluentd + become: true + ignore_errors: yes + shell: "kill -9 {{ item }}" + with_items: "{{ fluentdps.stdout_lines }}" + when: fluentdps.stdout_lines is defined + +# - fail: msg="this play requires fluentdps" +# when: fluentdps is not defined + + - name: ls fluentdps + debug: var=fluentdps.stdout + + # # ------------------------ + # # save variable > /tmp + # # ------------------------- + # - name: echo kill > tmp + # shell: "echo {{ fluentdps.stdout }} > /tmp/123" + # when: not fluentdps + # + +# -------------------------------------------------------------------------------------- +# --------------------- start and save ps tp tmp +# -------------------------------------------------------------------------------------- + + # ------------------------ + # start fluentd + # ------------------------- + - name: start fluentd background + shell: nohup /home/docker/.gem/ruby/2.5.0/bin/fluentd -c /fluentd/etc/fluent.conf -vv /dev/null 2>&1 & + + # ------------------------ + # start fluentd + # ------------------------- + - name: find1 fluentd + shell: ps efw -opid,cmd -Cfluentd | pgrep -o fluentd + register: fluentdps1 + + - name: ls fluentdps1 + debug: var=fluentdps1.stdout + + # ------------------------ + # save variable > /tmp1 + # ------------------------- + - name: echo > tmp1 + shell: "echo {{ fluentdps1.stdout }} > /tmp/12345" + + # ------------------------ + # example4net tcpdump example + # ------------------------- +# - name: google.com +# become: yes +# become_user: "{{ user }}" +# command: curl http://www.google.com +# ignore_errors: yes +# register: configwww +# +# - name: ls configwww +# debug: var=configwww.stdout_lines +# +# - name: ls -al /var/lab/playground/playground-readmongo/ +# become: yes +# become_user: "{{ user }}" +# #command: ls -al /var/lab/playground/playground-readmongo +# command: netstat -antlupe +# ignore_errors: yes +# register: config +# +# - name: ls config +# debug: var=config.stdout_lines +# +# - name: Refresh connection +# meta: clear_host_errors +# + diff --git a/fluentd/fluentd-config-update.yml.sh b/fluentd/fluentd-config-update.yml.sh new file mode 100644 index 0000000..4b8bf73 --- /dev/null +++ b/fluentd/fluentd-config-update.yml.sh @@ -0,0 +1,15 @@ +#!/bin/sh + + +ip4=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) +ip6=$(/sbin/ip -o -6 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + +echo "[service]" > /project/courses/fluentd/inventory.yml +/project/bin/swarmlab-nmap >> /project/courses/fluentd/inventory.yml + + +# include master or not +echo $ip4 >> /project/courses/fluentd/inventory.yml + + +ansible-playbook -u docker -i inventory.yml fluentd-config-update.yml -f 5 --ask-pass --ask-become-pass diff --git a/fluentd/fluentd-test-mongo.yml b/fluentd/fluentd-test-mongo.yml new file mode 100644 index 0000000..462548a --- /dev/null +++ b/fluentd/fluentd-test-mongo.yml @@ -0,0 +1,54 @@ +--- +- hosts: service + remote_user: docker + gather_facts: no + vars: + user: "docker" + + tasks: + +# -------------------------------------------------------------------------------------- +# --------------------- create test dir +# -------------------------------------------------------------------------------------- + + # ------------------------ + # test dir + # ------------------------- + - name: make dir for test + become: true + file: + path: "/var/log-in/test" + state: directory + owner: docker + group: docker + mode: '0755' + + - name: make dir2 + become: true + file: + path: "/var/log-in/test1" + state: directory + owner: docker + group: docker + mode: '0755' + +# -------------------------------------------------------------------------------------- +# --------------------- kill and save ps tp tmp +# -------------------------------------------------------------------------------------- + + - name: find fluentd + #shell: df -h >> /var/log-in/test/test + shell: df -h + #shell: "ps efw -opid,cmd -Cfluentd | pgrep -o fluentd" + register: fluentddate + + - name: ls fluentddate + debug: var=fluentddate.stdout_lines + + - name: write to /var/log-in/test/test2 + shell: "echo {{ item }} >> /var/log-in/test/test2" + with_items: "{{ fluentddate.stdout_lines }}" + + - name: write to /var/log-in/test1/test11 + shell: "echo {{ item }} >> /var/log-in/test1/test11" + with_items: "{{ fluentddate.stdout_lines }}" diff --git a/fluentd/fluentd.yml b/fluentd/fluentd.yml new file mode 100644 index 0000000..1894454 --- /dev/null +++ b/fluentd/fluentd.yml @@ -0,0 +1,198 @@ +--- +- hosts: service + remote_user: docker + gather_facts: no + vars: + user: "docker" + + tasks: + + # ------------------------ + # apt update + # ------------------------- + - name: apt update packages + become: true + apt: + update_cache: 'yes' + force_apt_get: 'yes' + upgrade: 'dist' + cache_valid_time: 3600 + install_recommends: true + autoremove: true + + # ------------------------ + # apt install packages + # ------------------------- + - name: apt install packages + become: true + apt: + update_cache: 'yes' + force_apt_get: 'yes' + install_recommends: true + autoremove: true + name: "{{ packages }}" + vars: + packages: + - build-essential + - git + - flex + - bison + - traceroute + - curl + - lynx + - ruby + - ruby-dev + + # ------------------------ + # directory4example fluentd + # ------------------------- + - name: make /var/log-in + become: true + file: + path: "/var/log-in" + state: directory + owner: docker + group: docker + mode: '0777' + + # ------------------------ + # gem begin + # ------------------------- + - name: make dir for gem + become: true + file: + path: "/home/docker/.gem" + state: directory + owner: docker + group: docker + mode: '0755' + + - name: gem install fluentd + #become: true + gem: + name: fluentd + version: 1.12.0 + state: present + environment: + CONFIGURE_OPTS: '--disable-install-doc' + PATH: '/home/docker/.gem/ruby/2.5.0/bin:{{ ansible_env.PATH }}' + + - name: gem install fluent-plugin-mongo + #become: true + gem: + name: fluent-plugin-mongo + state: present + + - name: gem install oj + #become: true + gem: + name: oj + state: present + + - name: gem install json + #become: true + gem: + name: json + state: present + + - name: gem install async-http + #become: true + gem: + name: async-http + version: 0.54.0 + state: present + + - name: gem install ext-monitor + #become: true + gem: + name: ext_monitor + version: 0.1.2 + state: present + + # ------------------------ + # gem end + # ------------------------- + + # ------------------------ + # add group + # ------------------------- +# - name: add group fluent +# become: true +# group: +# name: fluent +# state: present +# + # ------------------------ + # add user + # ------------------------- +# - name: add user gem +# become: true +# user: +# name: fluent +# group: fluent + + # ------------------------ + # mkdir directory4 fluent + # ------------------------- + - name: make dir fluentd + become: true + file: + path: "/fluentd/etc" + state: directory + owner: docker + group: docker + mode: '0755' + + - name: make dir fluentd + become: true + file: + path: "/fluentd/plugins" + state: directory + owner: docker + group: docker + mode: '0755' + + # ------------------------ + # cp fluentd.conf + # ------------------------- + - name: cp fluentd.conf + become: true + copy: + src: "./files/fluent.conf" + dest: /fluentd/etc/fluent.conf + owner: docker + group: docker + mode: 0755 + + # ------------------------ + # start fluentd + # ------------------------- + - name: start fluentd background + shell: nohup /home/docker/.gem/ruby/2.5.0/bin/fluentd -c /fluentd/etc/fluent.conf -vv /dev/null 2>&1 & + + # ------------------------ + # example4net tcpdump example + # ------------------------- +# - name: google.com +# become: yes +# become_user: "{{ user }}" +# command: curl http://www.google.com +# ignore_errors: yes +# register: configwww +# +# - name: ls configwww +# debug: var=configwww.stdout_lines +# +# - name: ls -al /var/lab/playground/playground-readmongo/ +# become: yes +# become_user: "{{ user }}" +# #command: ls -al /var/lab/playground/playground-readmongo +# command: netstat -antlupe +# ignore_errors: yes +# register: config +# +# - name: ls config +# debug: var=config.stdout_lines +# +# - name: Refresh connection +# meta: clear_host_errors diff --git a/fluentd/fluentd.yml.sh b/fluentd/fluentd.yml.sh new file mode 100644 index 0000000..16f84d1 --- /dev/null +++ b/fluentd/fluentd.yml.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +sudo apt update -y +sudo apt install -y ansible sshpass + +sudo mkdir -p /home/docker/.ansible +sudo chown docker.docker -R /home/docker + +sudo cp files/ansible.cfg /etc/ansible/ansible.cfg + +ip4=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) +ip6=$(/sbin/ip -o -6 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + +echo "[service]" > /project/courses/fluentd/inventory.yml +/project/bin/swarmlab-nmap >> /project/courses/fluentd/inventory.yml + + +# include master or not +echo $ip4 >> /project/courses/fluentd/inventory.yml + + +ansible-playbook -u docker -i inventory.yml fluentd.yml -f 5 --ask-pass --ask-become-pass +# 1st make sudo without password +# run with keys +#ansible-playbook -u docker -i inventory.yml fluentd.yml -f 5 --private-key=/home/docker/.ssh/id_rsa