From a17f7f8838efb94085602ce005ca4d1fae72c9d2 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 11 Jun 2020 17:53:16 +0000 Subject: [PATCH] Larger changes --- DOC.md | 3 +++ Makefile | 7 ++----- README.md | 3 +++ ansible.cfg | 3 ++- roles/lxc_swarm/tasks/deploy.yml | 3 +++ roles/lxc_swarm/tasks/lxc_nodes.yml | 4 ++++ roles/lxc_swarm/tasks/main.yml | 4 ++++ roles/lxc_swarm/tasks/swarm.yml | 3 ++- roles/lxc_swarm/templates/Dockerfile.j2 | 4 ++-- roles/lxc_swarm/templates/galera.cnf.j2 | 5 ++++- roles/lxc_swarm/vars/main.yml | 5 ++++- stack.yaml | 26 +++++++++++++++++++++---- 12 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 roles/lxc_swarm/tasks/deploy.yml diff --git a/DOC.md b/DOC.md index 692cad0..5421389 100644 --- a/DOC.md +++ b/DOC.md @@ -3,3 +3,6 @@ Command for checking the docker node status: ```docker node ls``` Command for using and attaching to LXC container (e.g. to 'worker1'): ```lxc-attach --name worker1```. + +One possible limitation of this process, is that static ip assignments [are missing in docker +swarm](https://forums.docker.com/t/docker-swarm-1-13-static-ips-for-containers/28060/13). diff --git a/Makefile b/Makefile index d02a109..07a29e0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: preautomation provision config swarm_deploy +all: preautomation provision swarm_deploy preautomation: /bin/bash ./ansible_install.sh @@ -6,8 +6,5 @@ preautomation: provision: /bin/bash ./ansible_provisioning.sh -config: - /bin/bash ./ansible_swarm_config.sh - swarm_deploy: - docker stack deploy -c swarm-galera.yml stack + /bin/bash ./ansible_swarm_config.sh diff --git a/README.md b/README.md index ac3b06f..5a7f52f 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ * Linux Containers (LXC) for swarm worker nodes * Ansible for automatic deployment * Galera-inside-docker for MariaDB clustering + +As an operating system we used the same for all docker nodes (Host machine & LXC containers): + * Debian 9.3 (Stretch) with Linux Kernel 4.9.0-5-amd64 In more detail, the manager node is running on the host machine itself, while the provisioning of LXC containers permits the creation of multiple worker nodes on which the galera cluster runs, running on the same host without the need for separate worker machines. diff --git a/ansible.cfg b/ansible.cfg index a706507..0a455bc 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,4 +1,5 @@ [defaults] #added parameters for actual management of LXC nodes with ansible because of bridged networking packet loss gather_timeout = 30 -timeout = 30 +timeout = 45 +host_key_checking = False diff --git a/roles/lxc_swarm/tasks/deploy.yml b/roles/lxc_swarm/tasks/deploy.yml new file mode 100644 index 0000000..4909165 --- /dev/null +++ b/roles/lxc_swarm/tasks/deploy.yml @@ -0,0 +1,3 @@ +--- +- name: Deploy swarm stack + shell: docker stack deploy -c stack.yaml stack diff --git a/roles/lxc_swarm/tasks/lxc_nodes.yml b/roles/lxc_swarm/tasks/lxc_nodes.yml index 86d56e9..1fccfe3 100644 --- a/roles/lxc_swarm/tasks/lxc_nodes.yml +++ b/roles/lxc_swarm/tasks/lxc_nodes.yml @@ -18,6 +18,10 @@ - "worker2" when: inventory_hostname in groups['manager'] +- name: Pause execution for 20 seconds + pause: + seconds: 20 + - name: Start lxc containers shell: lxc-start --name {{ item }} with_items: diff --git a/roles/lxc_swarm/tasks/main.yml b/roles/lxc_swarm/tasks/main.yml index 79a6ebb..0f86e24 100644 --- a/roles/lxc_swarm/tasks/main.yml +++ b/roles/lxc_swarm/tasks/main.yml @@ -20,3 +20,7 @@ - include: Dockerfile.yml when: inventory_hostname in groups['workers'] tags: swarm + +- include: deploy.yml + when: inventory_hostname in groups['manager'] + tags: swarm diff --git a/roles/lxc_swarm/tasks/swarm.yml b/roles/lxc_swarm/tasks/swarm.yml index ebdcc0a..dd263f3 100644 --- a/roles/lxc_swarm/tasks/swarm.yml +++ b/roles/lxc_swarm/tasks/swarm.yml @@ -23,7 +23,8 @@ file: path: /var/container_data/mysql state: directory - mode: '0755' + mode: '0777' + recurse: true when: inventory_hostname in groups['workers'] - name: Add docker labels for galera master and secondary nodes diff --git a/roles/lxc_swarm/templates/Dockerfile.j2 b/roles/lxc_swarm/templates/Dockerfile.j2 index 731bfcb..ed9b222 100644 --- a/roles/lxc_swarm/templates/Dockerfile.j2 +++ b/roles/lxc_swarm/templates/Dockerfile.j2 @@ -6,6 +6,6 @@ RUN add-apt-repository "deb [arch=amd64,arm64,i386,ppc64el] http://mirrors.corei RUN apt-get update RUN apt install -y rsync mariadb-server-10.4 galera-4 mariadb-client-10.4 RUN systemctl unmask mariadb -COPY /opt/galera.cnf /etc/mysql/conf.d/galera.cnf -RUN echo "wsrep_node_address=`ip a s eth0|grep inet|head -1|awk '{print $2}'|cut -d "/" -f 1`" >> /etc/mysql/conf.d/galera.cnf +COPY galera.cnf /etc/mysql/conf.d/galera.cnf +#RUN echo "wsrep_node_address=`ip a s eth0|grep inet|head -1|awk '{print $2}'|cut -d "/" -f 1`" >> /etc/mysql/conf.d/galera.cnf ENTRYPOINT ["mysqld","{{ extra_docker_options }}"] diff --git a/roles/lxc_swarm/templates/galera.cnf.j2 b/roles/lxc_swarm/templates/galera.cnf.j2 index dfc6bb6..eb23f1a 100644 --- a/roles/lxc_swarm/templates/galera.cnf.j2 +++ b/roles/lxc_swarm/templates/galera.cnf.j2 @@ -10,7 +10,8 @@ wsrep_provider=/usr/lib/galera/libgalera_smm.so # Galera Cluster Configuration wsrep_cluster_name="galera_cluster" -wsrep_cluster_address="gcomm://{% for host in groups['workers'] %} {{hostvars[host]['ansible_default_ipv4']['address']}} {% endfor %}" +#wsrep_cluster_address="gcomm://{% for host in groups['workers'] %} {{hostvars[host]['ansible_default_ipv4']['address']}} {% endfor %}" +wsrep_cluster_address="gcomm://{% for address in container_address %} {{address}} {% endfor %}" # Galera Synchronization Configuration wsrep_sst_method=rsync @@ -18,4 +19,6 @@ wsrep_sst_method=rsync # Galera Node Configuration #wsrep_node_address="{{ ansible_default_ipv4.address }}" #wsrep_node_name="{{ ansible_hostname }}" + +wsrep_node_address="{{ container_address[0] if ansible_hostname in groups['workers'][0] else container_address[1] }} wsrep_node_name="{{ nodes[0] if ansible_hostname in groups['workers'][0] else nodes[1] }}" diff --git a/roles/lxc_swarm/vars/main.yml b/roles/lxc_swarm/vars/main.yml index 6c0427c..d3749a8 100644 --- a/roles/lxc_swarm/vars/main.yml +++ b/roles/lxc_swarm/vars/main.yml @@ -1,5 +1,5 @@ interface: lxcbr0 -extra_docker_options: "{{ ' --wsrep-new-cluster' if ansible_hostname == 'worker1' else '' }}" +extra_docker_options: "{{ '--wsrep-new-cluster' if ansible_hostname == 'worker1' else '' }}" workers: - "10.0.3.100" - "10.0.3.101" @@ -7,3 +7,6 @@ workers: nodes: - "node1" - "node2" +container_address: + - "10.0.3.10" + - "10.0.3.11" diff --git a/stack.yaml b/stack.yaml index 50039d1..8ca6111 100644 --- a/stack.yaml +++ b/stack.yaml @@ -14,22 +14,27 @@ services: # - TASKID={{.Task.ID}} # - TASKNAME={{.Task.Name}} # - TASKREPID={{.Task.Slot}} + hostname: node1 + # Storage volume volumes: - /var/container_data/mysql:/var/lib/mysql networks: galera_net: - + ipv4_address: 10.0.3.10 deploy: + # Replicated mode + mode: replicated replicas: 1 restart_policy: delay: 10s max_attempts: 10 window: 60s placement: + # Services must run only on workers with one service per container constraints: - node.labels.node1 == true - node.role == worker @@ -54,26 +59,39 @@ services: image: ubuntu:galera-node-worker2 +# environment: +# - NODENAME={{.Node.Hostname}} +# - NODEID={{.Node.ID}} +# - SERVICEID={{.Service.ID}} +# - SERVICENAME={{.Service.Name}} +# - TASKID={{.Task.ID}} +# - TASKNAME={{.Task.Name}} +# - TASKREPID={{.Task.Slot}} + hostname: node2 + # Storage volume volumes: - /var/container_data/mysql:/var/lib/mysql networks: galera_net: - + ipv4_address: 10.0.3.11 deploy: + # Replicated mode + mode: replicated replicas: 1 restart_policy: delay: 10s max_attempts: 10 window: 60s placement: + # Services must run only on workers with one service per container constraints: - node.labels.node2 == true - - node.role == worker -# - node.group == galera + - node.role == worker +# - node.group == galera depends_on: - db_cluster_node1 ports: