swarmlab
/
swarmlab-microservice-example
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
15 Commits
1 Branch
0 Tags
1.1 MiB
 
 
Tree: 4a083a9d64
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '4a083a9d64'
${ noResults }
swarmlab-microservice-example/install/usr/share/swarmlab.io/sec/swarmlab-sec

776 lines
19 KiB
Raw Blame History

#!/bin/bash
# The MIT License (MIT)
#
# rootApostolos@swarmlab.io
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# Origin: https://github.com/NLKNguyen/alpine-mpich
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
TARGET="$(readlink "$SOURCE")"
if [[ $TARGET == /* ]]; then
SOURCE="$TARGET"
else
DIR="$( dirname "$SOURCE" )"
SOURCE="$DIR/$TARGET" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
fi
done
SRPATH="$( dirname "$SOURCE" )"
SFPATH="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
if [ "$SFPATH" != "$SRPATH" ]; then
RDIR=$SRPATH # relativ path directory
fi
cwdir=$PWD
set -e
# edit it - EDIT-BEGIN
# ##############################################
#
# Rules For Naming
#
# prefix must be one of the following
#
# For microservice
# microservice-
#
# For poc
# poc-
#
# For storage
# storage-
#
# For Labroom
# hybrid-
#
#
# You can use ONLY lowercase letters and numbers.
#
# ##############################################
. $SRPATH/swarmlab-template-image
DOCKERuser="docker"
Wdir=${cwdir}/$IMAGE_local
if [ ! -d $Wdir ]; then
mkdir -p $Wdir
fi
APTPARAM=" --no-install-recommends "
HYBRID_NETWORK=$IMAGE_local
bootstrap="sec_bootstrap"
hostnames="auto_update_hosts"
hostnames_get="get_hosts"
NODENAME=${IMAGE_local}_masterservice_1.${IMAGE_local}_${HYBRID_NETWORK}
NODENETWORK=${HYBRID_NETWORK}
# -----------------------------------------------
#
# Load Default config swarmlab.io
#
# ----------------------------------------------
function EPHEMERAL_PORT() {
LOW_BOUND=49152
RANGE=16384
while true; do
CANDIDATE=$[$LOW_BOUND + ($RANDOM % $RANGE)]
(echo "" >/dev/tcp/127.0.0.1/${CANDIDATE}) >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo $CANDIDATE
break
fi
done
}
servicesshport=$(EPHEMERAL_PORT)
if [ ! -f $Wdir/.env ]; then
cat << EOF > $Wdir/.env
REGISTRY_ADDR=localhost
REGISTRY_PORT=5000
IMAGE_NAME=$HYBRID_NETWORK
SSH_PORT=$serviceshport
R_PORT1=$(EPHEMERAL_PORT)
R_PORT2=$(EPHEMERAL_PORT)
R_PORT3=$(EPHEMERAL_PORT)
R_PORT4=$(EPHEMERAL_PORT)
R_PORT5=$(EPHEMERAL_PORT)
EOF
fi
. $Wdir/.env
# you do not need to edit it - END
create_dockerfile ()
{
docker pull $IMAGE_origin << ANSWERS
yes
yes
yes
ANSWERS
. $Wdir/ROOT_PASSWORD
if [ -d "$Wdir/project" ]; then
# -----------------------------------------------
#
# create Dockerfile
#
# ----------------------------------------------
search='ok'
if [ $search == 'ok' ]
then
echo ""
echo ">>> Load Origin "
cat << EOF > $Wdir/Dockerfile
FROM $IMAGE_origin
#
USER root
COPY $bootstrap /usr/bin
COPY $hostnames_get /usr/bin
COPY $hostnames /usr/bin
COPY install-vim-plugin.sh .
ENV NOTVISIBLE "in users profile"
ENV USER1 docker
ENV USER_HOME /home/docker
ENV SSHDIR \${USER_HOME}/.ssh
ENV PATH=/usr/local/texlive/bin/x86_64-linuxmusl:/usr/local/texlive/2020/bin/x86_64-linuxmusl:${PATH}
COPY ssh/ \${SSHDIR}/
COPY setup.sh \
extrapackages \
/
EOF
cat $Wdir/install/usr/share/swarmlab.io/sec/Dockerfile.tmp >> $Wdir/Dockerfile
fi
else
echo ""
echo "Not in Project Directory"
echo "A project directory should look like this"
echo ""
cat <<EOF
├── docker-compose.yml
├── Dockerfile
├── out.log
├── project
│   └── hello_world.sh
└── ssh
├── id_rsa
└── id_rsa.pub
EOF
echo "Change to your Project Directory And Try Again"
echo ""
exit 0
fi
}
create_project ()
{
docker pull $IMAGE_origin << ANSWERS
yes
yes
yes
ANSWERS
if [ "$Wdir" == "$HOME" ]; then
echo""
echo "You are in Your Home directory"
echo "Please create a project directory"
echo "mkdir myproject; cd myproject; swarmlab-sec create"
echo""
else
echo "# -----------------------------------------------"
echo "##### Create project #####"
echo "# -----------------------------------------------"
search='ok'
if [ $search == 'ok' ]
then
echo ">>> Load Origin "
cat << EOF > $Wdir/Dockerfile
FROM $IMAGE_origin
#
USER root
COPY $bootstrap /usr/bin
COPY $hostnames_get /usr/bin
COPY $hostnames /usr/bin
COPY install-vim-plugin.sh .
ENV NOTVISIBLE "in users profile"
ENV USER1 docker
ENV USER_HOME /home/docker
ENV SSHDIR \${USER_HOME}/.ssh
ENV PATH=/usr/local/texlive/bin/x86_64-linuxmusl:/usr/local/texlive/2020/bin/x86_64-linuxmusl:${PATH}
COPY ssh/ \${SSHDIR}/
COPY setup.sh \
extrapackages \
/
EOF
cat $Wdir/install/usr/share/swarmlab.io/sec/Dockerfile.tmp >> $Wdir/Dockerfile
fi
# you do not need to edit it - BEGIN
/bin/mkdir -p $Wdir/project
/bin/mkdir -p $Wdir/logs
/bin/cp -rf $SRPATH/project/bin $Wdir/project
/bin/cp -rf $SRPATH/project/config $Wdir/project
/bin/cp -f $SRPATH/$bootstrap $Wdir/$bootstrap
/bin/cp -f $SRPATH/$hostnames $Wdir/$hostnames
/bin/cp -f $SRPATH/$hostnames_get $Wdir/$hostnames_get
/bin/cp -f $SRPATH/ROOT_PASSWORD $Wdir/ROOT_PASSWORD
/bin/cp -rf $SRPATH/.vimrc $Wdir/.vimrc
/bin/cp -rf $SRPATH/extrapackages $Wdir
/bin/cp -rf $SRPATH/setup.sh $Wdir
/bin/cp -rf $SRPATH/fonts-extra $Wdir
/bin/cp -rf $SRPATH/docs $Wdir
/bin/cp -rf $SRPATH/run-gui.sh $Wdir/run-gui.sh
/bin/cp -rf $SRPATH/install-vim-plugin.sh $Wdir/install-vim-plugin.sh
# you do not need to edit it - END
# ###############################
# EDIT-BEGIN
# ###############################
/bin/cp -rf $SRPATH/cls $Wdir
/bin/cp -rf $SRPATH/latex-all $Wdir/latex-all
# ###############################
# EDIT-END
# ###############################
# you do not need to edit it - BEGIN
cat << FOE > $Wdir/swarmlab-index.js
export const myApiConfig = [
{
"swarmlabname": "$IMAGE_local",
"git": "$GIT_origin",
"gui": $service_GUI,
"web": $service_WEB,
"url": "$service_URL",
"version": 0,
"port": 0
}
];
FOE
cat << EOF > $Wdir/docker-compose.yml
version: "3"
services:
masterservice:
image: $IMAGE_NAME
privileged: true
environment:
- NODENAME=${NODENAME}
- NODENETWORK=${NODENETWORK}
- DISPLAY=\${DISPLAY}
cap_add:
- NET_ADMIN
# ###############################
# EDIT-BEGIN
#
# Open Ports you can use
# {R_PORT1} - {R_PORT5}
#
# ###############################
user: root
entrypoint: ["sec_bootstrap", "role=masterservice", "sec_master_service_name=masterservice", "sec_worker_service_name=workerservice"]
#ports:
# - "\${R_PORT1}:80"
# - "\${R_PORT2}:443"
networks:
- ${HYBRID_NETWORK}
volumes:
# #############################
# you do not need to edit it
#
# - $Wdir/project:/home/docker/project
# - $Wdir/$bootstrap:/usr/bin/$bootstrap
# #############################
- $Wdir/project:/home/docker/project
- $Wdir/$bootstrap:/usr/bin/$bootstrap
- $Wdir/latex-all:/usr/local/bin/latex-all
- $Wdir/docs:/home
- /tmp/.X11-unix:/tmp/.X11-unix
# ###############################
# EDIT-END
# ###############################
# workerservice:
# image: $IMAGE_NAME
# privileged: true
# environment:
# - NODENAME=${NODENAME}
# - NODENETWORK=${NODENETWORK}
# - DISPLAY=\${DISPLAY}
# cap_add:
# - NET_ADMIN
# user: root
# entrypoint: ["sec_bootstrap", "role=workerservice", "sec_master_service_name=masterservice", "sec_worker_service_name=workerservice"]
# #ports:
# # - "\${SSH_PORT}:22"
# networks:
# - ${HYBRID_NETWORK}
# volumes:
# - $Wdir/project:/home/docker/project
# - $Wdir/project/data-www:/data-www
# - $Wdir/project/config/nginx.conf:/etc/nginx/nginx.conf
# - $Wdir/project/config/default.conf:/etc/nginx/conf.d/default.conf
# - $Wdir/project/config/supervisord.conf:/etc/supervisor/supervisord.conf
networks:
${HYBRID_NETWORK}:
EOF
cat << EOF > $Wdir/stop.sh
../install/usr/share/swarmlab.io/sec/swarmlab-sec down
EOF
cat << EOF > $Wdir/container-stop.sh
docker stop \$1
docker container rm \$1
EOF
cat <<EOF
Project is Ready
├── docker-compose.yml
├── Dockerfile
├── out.log
├── project
│   └── hello_world.sh
└── ssh
├── id_rsa
└── id_rsa.pub
Run: swarmlab-sec up size=5
or run swarmlab-sec help for USAGE
EOF
/bin/chmod -R 777 $Wdir
/bin/chown -R sec.sec $Wdir
fi
}
#######################
# TASK INDICATORS
COMMAND_UP=0
COMMAND_CREATE=0
COMMAND_DOWN=0
COMMAND_RELOAD=0
COMMAND_SCALE=0
COMMAND_LOGIN=0
COMMAND_EXEC=0
COMMAND_LIST=0
COMMAND_CLEAN=0
# Default values if providing empty
SIZE=4
#############################################
usage ()
{
echo ""
echo "--------------"
echo " SwarmLab.io "
echo "--------------"
echo ""
echo " USAGE: ./swarmlab-sec [COMMAND] [OPTIONS]"
echo ""
echo " Examples of [COMMAND] can be:"
echo " create: create swarmlab-sec "
echo " mkdir WORKdir; cd WORKdir; swarmlab-sec create"
echo ""
echo " up: start swarmlab-sec"
echo " swarmlab-sec up size=10"
echo ""
echo " scale: resize the swarmlab-sec"
echo " swarmlab-sec scale size=30"
echo ""
echo " reload: rebuild image and distribute to nodes"
echo " swarmlab-sec reload size=15"
echo ""
echo " login: login to Docker container of MPI master node for interactive usage"
echo " swarmlab-sec login"
echo ""
echo " exec: execute shell command at the MPI master node"
echo " swarmlab-sec exec [SHELL COMMAND]"
echo ""
echo " down: shutdown swarmlab-sec"
echo " swarmlab-sec down"
echo ""
echo " clean: remove images in the system"
echo " swarmlab-sec clean"
echo ""
echo " list: show running containers of swarmlab-sec"
echo " swarmlab-sec list"
echo ""
echo " help: show this message"
echo " swarmlab-sec help"
echo ""
echo " "
}
HEADER="
SwarmLab.io"
clear_all ()
{
$(docker stop $(docker ps -a | grep "${IMAGE_local}_masterservice_" | awk '{print $1}')) || true
$(docker stop $(docker ps -a | grep "${IMAGE_local}_workerservice_" | awk '{print $1}')) || true
#$(docker stop $(docker ps -a | grep "_registry_" | awk '{print $1}')) || true # remove for microservices
#docker ps -a | grep "_registry_" | awk '{print $1}' | xargs docker container rm || true # remove for microservices
docker ps -a | grep "${IMAGE_local}_workerservice_" | awk '{print $1}' | xargs docker container rm || true
docker ps -a | grep "${IMAGE_local}_masterservice_" | awk '{print $1}' | xargs docker container rm || true
docker rmi $IMAGE_local -f
}
down_all ()
{
printf "\\n\\n===> CLEAN UP SWARMLAB"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose down"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml down
}
up_registry ()
{
printf "\\n\\n===> SPIN UP REGISTRY"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose up -d registry"
printf "\\n"
#docker stop swarmlab-registry || true && docker rm swarmlab-registry || true # remove for microservices
docker container prune --force
docker-compose -f $Wdir/docker-compose.yml up --no-recreate -d registry
}
generate_ssh_keys ()
{
if [ -f ssh/id_rsa ] && [ -f ssh/id_rsa.pub ]; then
return 0
fi
printf "\\n\\n===> GENERATE SSH KEYS \\n\\n"
echo "$ mkdir -p ssh/ "
printf "\\n"
cd $Wdir
mkdir -p ssh/
echo "$ ssh-keygen -f ssh/id_rsa -t rsa -N ''"
printf "\\n"
ssh-keygen -f ssh/id_rsa -t rsa -N ''
cd $cwdir
}
build_and_push_image ()
{
printf "\\n\\n===> BUILD IMAGE"
printf "\\n%s\\n" "$HEADER"
echo "$ docker build -t \"$IMAGE_NAME\" ."
printf "\\n"
#docker build -t "$REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME" .
cd $Wdir
docker build --force-rm --pull -t "$IMAGE_NAME" .
cd $cwdir
}
up_master ()
{
printf "\\n\\n===> SPIN UP MASTER NODE"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose up -d masterservice"
printf "\\n"
echo "$ $IMAGE_local -d $IMAGE_origin"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml rm -f -s -v
docker-compose -f $Wdir/docker-compose.yml up --build --remove-orphans --force-recreate -d masterservice << ANSWERS
yes
yes
yes
ANSWERS
#docker-compose up --force-recreate -d masterservice
}
up_workers ()
{
printf "\\n\\n===> SPIN UP WORKER NODES"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose up -d worker"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml rm -f -s -v
docker-compose -f $Wdir/docker-compose.yml up --build --force-recreate --renew-anon-volumes --remove-orphans -d workerservice
#docker-compose up --force-recreate -d workerservice
printf "\\n"
printf "\\n%s\\n" "$HEADER"
NUM_WORKER=$((SIZE - 1))
echo "$ docker-compose scale workerservice=$NUM_WORKER"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml scale workerservice=${NUM_WORKER}
}
down_master ()
{
printf "\\n\\n===> TORN DOWN MASTER NODE"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose stop masterservice && docker-compose rm -f masterservice"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml stop masterservice && docker-compose -f $Wdir/docker-compose.yml rm -f masterservice
}
down_workers ()
{
printf "\\n\\n===> TORN DOWN WORKER NODES"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose stop worker && docker-compose rm -f worker"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml stop workerservice && docker-compose -f $Wdir/docker-compose.yml rm -f workerservice
}
list ()
{
printf "\\n\\n===> LIST CONTAINERS"
printf "\\n%s\\n" "$HEADER"
echo "$ docker-compose ps"
printf "\\n"
docker-compose -f $Wdir/docker-compose.yml ps
}
exec_on_mpi_master_container ()
{
# shellcheck disable=SC2046
docker exec -it -u $DOCKERuser $(docker-compose -f $Wdir/docker-compose.yml ps | grep 'masterservice'| awk 'NR==1{print $1}') "$@"
}
prompt_ready ()
{
printf "\\n\\n===> SWARMLAB READY \\n\\n"
}
show_instruction ()
{
echo ' ## . '
echo ' ## ## ## == '
echo ' ## ## ## ## ## === '
echo ' /"""""""""""""""""\___/ === '
echo ' ~~~ {~~ ~~~~ ~~~ ~~~~ ~~~ ~ / ===- ~~~ '
echo ' \______ o __/ '
echo ' \ \ __/ '
echo ' \____\_______/ '
echo ' '
echo ' Swarmlab.io Hybrid '
echo ''
echo '=============================================================='
echo ''
echo "To run SEC programs in an interative shell:"
echo " 1. Login to masterservice node:"
echo " Using Container->connect Menou:"
echo " copy/paste and Run command"
echo ""
echo " Or using SSH with keys through exposed port:"
echo " $ ssh -o \"StrictHostKeyChecking no\" -i ssh/id_rsa -p $SSH_PORT docker@localhost"
echo ' where [localhost] could be changed to the host IP of masterservice node'
echo ""
echo " 2. Execute programs inside masterservice node, for example:"
echo " $ sudo su"
echo " # apk update"
echo " *----------------------------------------------------*"
echo " | Default hostfile of connected nodes in the swarmlab |"
echo " | To obtain hostfile manually: $ ./bin/swarmlab-nmap: > hosts |"
echo " * ---------------------------------------------------*"
echo ""
echo ""
}
#############################################
while [ "$1" != "" ];
do
PARAM=$(echo "$1" | awk -F= '{print $1}')
VALUE=$(echo "$1" | awk -F= '{print $2}')
case $PARAM in
help)
usage
exit
;;
-i)
show_instruction
exit
;;
login)
COMMAND_LOGIN=1
;;
exec)
COMMAND_EXEC=1
shift # the rest is the shell command to run in the node
SHELL_COMMAND="$*"
break # end while loop
;;
up)
COMMAND_UP=1
;;
create)
COMMAND_CREATE=1
;;
down)
COMMAND_DOWN=1
;;
reload)
COMMAND_RELOAD=1
;;
scale)
COMMAND_SCALE=1
;;
list)
COMMAND_LIST=1
;;
clean)
COMMAND_CLEAN=1
;;
size)
[ "$VALUE" ] && SIZE=$VALUE
;;
*)
echo "ERROR: unknown parameter \"$PARAM\""
usage
exit 1
;;
esac
shift
done
if [ $COMMAND_UP -eq 1 ]; then
create_dockerfile
down_all
clear_all
#up_registry # remove for microservices
generate_ssh_keys
build_and_push_image # remove for microservices
up_master
#up_workers
prompt_ready
show_instruction
elif [ $COMMAND_CREATE -eq 1 ]; then
create_project
elif [ $COMMAND_DOWN -eq 1 ]; then
down_all
clear_all
elif [ $COMMAND_CLEAN -eq 1 ]; then
clear_all
elif [ $COMMAND_SCALE -eq 1 ]; then
create_dockerfile
down_master
down_workers
up_master
#up_workers
prompt_ready
show_instruction
elif [ $COMMAND_RELOAD -eq 1 ]; then
create_dockerfile
down_master
down_workers
build_and_push_image
up_master
#up_workers
prompt_ready
show_instruction
elif [ $COMMAND_LOGIN -eq 1 ]; then
exec_on_mpi_master_container /bin/bash
elif [ $COMMAND_EXEC -eq 1 ]; then
create_dockerfile
exec_on_mpi_master_container ash -c "${SHELL_COMMAND}"
elif [ $COMMAND_LIST -eq 1 ]; then
list
else
usage
fi