hybrid-mpi/install/usr/share/swarmlab.io/sec/swarmlab-mpi

#!/bin/bash

# The MIT License (MIT)
#
# rootApostolos@swarmlab.io
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Origin: https://github.com/NLKNguyen/alpine-mpich

set -e

# shellcheck disable=SC1091
#. ./.env


# -----------------------------------------------
# 
#  Find Source path of swarmlab.io script
#
# ----------------------------------------------


SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
  SOURCE="$(readlink "$SOURCE")"
  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"


# -----------------------------------------------
# 
#  Load Default config swarmlab.io 
#
# ----------------------------------------------


. $DIR/.env


# -----------------------------------------------
# 
#  Find Working dir 
#
# ----------------------------------------------


Wdir=$(pwd)


create_dockerfile ()
{
if [ -d "$Wdir/project" ]; then
# -----------------------------------------------
# 
#  create Dockerfile
#
# ----------------------------------------------

rm -f $Wdir/out.log 2> /dev/null

error="Error:";
IMAGE=$(docker inspect --type=image $REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME:latest 2>&1 | tee $Wdir/out.log)
if search=$(grep "$error" $Wdir/out.log)
then
        echo ""
	echo ">>>	Load Origin 	"
cat << EOF > $Wdir/Dockerfile
FROM nlknguyen/alpine-mpich:onbuild
WORKDIR /project
EOF
else
	echo ">>>	Load Local 	"
cat << EOF > $Wdir/Dockerfile
FROM localhost:5000/mpi
WORKDIR /project
EOF
fi
else
	echo ""
	echo "Not in Project Directory"
	echo "A  project directory should look like this"
	echo ""

cat <<EOF 
├── docker-compose.yml
├── Dockerfile
├── out.log
├── project
│   └── mpi_hello_world.c
└── ssh
    ├── id_rsa
    └── id_rsa.pub
EOF
	echo "Change to your Project Directory And Try Again"
        echo ""
exit 0
fi
}

create_project ()
{

if [ "$Wdir" ==  "$HOME" ]; then
        echo""
	echo "You are in Your Home directory"
	echo "Please create a project directory"
	echo "mkdir myproject; cd myproject; swarmlab-mpi create"
        echo""
else
echo "# -----------------------------------------------"
echo "#####  		Create project            #####"
echo "# -----------------------------------------------"

# clean old images
#docker rmi $(docker images -f "dangling=true" -q)

rm -f $Wdir/out.log 2> /dev/null

error="Error:";
IMAGE=$(docker inspect --type=image $REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME:latest 2>&1 | tee $Wdir/out.log)
if search=$(grep "$error" $Wdir/out.log)
then
	echo ">>>	Load Origin 	"
cat << EOF > $Wdir/Dockerfile
FROM nlknguyen/alpine-mpich:onbuild
WORKDIR /project
CMD export PATH=$PATH:/project
EOF
else
	echo ">>>	Load Local 	"
cat << EOF > $Wdir/Dockerfile
FROM localhost:5000/mpi
WORKDIR /project
CMD export PATH=$PATH:/project
EOF
fi


/bin/mkdir -p $Wdir/project
/bin/cp -f $DIR/project/mpi_hello_world.c $Wdir/project


cat << EOF > $Wdir/docker-compose.yml
version: "2"

services:
  registry:
    image: registry
    #container_name: swarmlab-registry
    ports:
      - "${REGISTRY_PORT}:5000"

  master:
    image: $REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME
    #container_name: swarmlab-mpi
    user: root
    entrypoint: ["mpi_bootstrap", "role=master", "mpi_master_service_name=master", "mpi_worker_service_name=worker"]
    ports:
      - "${SSH_PORT}:22"
    networks:
      - net
    volumes:
      - $Wdir/project:/project


  worker:
    image: $REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME
    user: root
    entrypoint: ["mpi_bootstrap", "role=worker", "mpi_master_service_name=master", "mpi_worker_service_name=worker"]
    networks:
      - net
    volumes:
      - $Wdir/project:/project

networks:
  net:
EOF

/bin/cp -rf $DIR/ssh $Wdir

cat <<EOF 

Project is Ready
├── docker-compose.yml
├── Dockerfile
├── out.log
├── project
│   └── mpi_hello_world.c
└── ssh
    ├── id_rsa
    └── id_rsa.pub

Run: swarmlab-mpi up size=5

     or run swarmlab-mpi help for USAGE

EOF

/bin/chmod -R 777 $Wdir
/bin/chown -R  mpi.mpi  $Wdir

fi
}

#######################
# TASK INDICATORS
COMMAND_UP=0
COMMAND_CREATE=0
COMMAND_DOWN=0
COMMAND_RELOAD=0
COMMAND_SCALE=0
COMMAND_LOGIN=0
COMMAND_EXEC=0
COMMAND_LIST=0
COMMAND_CLEAN=0

# Default values if providing empty
SIZE=4

#############################################
usage ()
{
    echo ""
    echo "--------------"
    echo " SwarmLab.io "
    echo "--------------"
    echo ""
    echo " USAGE: ./swarmlab-mpi [COMMAND] [OPTIONS]"
    echo ""
    echo " Examples of [COMMAND] can be:"
    echo "      create: create swarmlab-mpi "
    echo "          mkdir WORKdir; cd WORKdir; swarmlab-mpi create"
    echo ""
    echo "      up: start swarmlab-mpi"
    echo "          swarmlab-mpi up size=10"
    echo ""
    echo "      scale: resize the swarmlab-mpi"
    echo "          swarmlab-mpi scale size=30"
    echo ""
    echo "      reload: rebuild image and distribute to nodes"
    echo "          swarmlab-mpi reload size=15"
    echo ""
    echo "      login: login to Docker container of MPI master node for interactive usage"
    echo "          swarmlab-mpi login"
    echo ""
    echo "      exec: execute shell command at the MPI master node"
    echo "          swarmlab-mpi exec [SHELL COMMAND]"
    echo ""
    echo "      down: shutdown swarmlab-mpi"
    echo "          swarmlab-mpi down"
    echo ""
    echo "      clean: remove images in the system"
    echo "          swarmlab-mpi clean"
    echo ""
    echo "      list: show running containers of swarmlab-mpi"
    echo "          swarmlab-mpi list"
    echo ""
    echo "      help: show this message"
    echo "          swarmlab-mpi help"
    echo ""
    echo "  "
}

HEADER="
SwarmLab.io"

clear_all ()
{
	$(docker stop $(docker ps -a | grep "_master_" | awk '{print $1}')) || true
	$(docker stop $(docker ps -a | grep "_worker_" | awk '{print $1}')) || true
	$(docker stop $(docker ps -a | grep "_registry_" | awk '{print $1}')) || true
	docker ps -a | grep "_registry_" | awk '{print $1}' | xargs docker container rm || true
	docker ps -a | grep "_worker_" | awk '{print $1}' | xargs docker container rm || true
	docker ps -a | grep "_master_" | awk '{print $1}' | xargs docker container rm || true
}

down_all ()
{
    printf "\\n\\n===> CLEAN UP SWARMLAB"

    printf "\\n%s\\n" "$HEADER"
    echo "$ docker-compose down"
    printf "\\n"

    #docker stop swarmlab-mpi || true && docker rm swarmlab-mpi || true
    docker-compose down
}

up_registry ()
{
    printf "\\n\\n===> SPIN UP REGISTRY"

    printf "\\n%s\\n" "$HEADER"
    echo "$ docker-compose up -d registry"
    printf "\\n"

    #docker stop swarmlab-registry || true && docker rm swarmlab-registry || true
    docker container prune --force
    docker-compose up --no-recreate -d registry
}

generate_ssh_keys ()
{
    if [ -f ssh/id_rsa ] && [ -f ssh/id_rsa.pub ]; then
        return 0
    fi

    printf "\\n\\n===> GENERATE SSH KEYS \\n\\n"

    echo "$ mkdir -p ssh/ "
    printf "\\n"
    mkdir -p ssh/

    echo "$ ssh-keygen -f ssh/id_rsa -t rsa -N ''"
    printf "\\n"
    ssh-keygen -f ssh/id_rsa -t rsa -N ''
}

build_and_push_image ()
{
    printf "\\n\\n===> BUILD IMAGE"
    printf "\\n%s\\n" "$HEADER"
    echo "$ docker build -t \"$REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME\" ."
    printf "\\n"
    docker build -t "$REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME" .

    printf "\\n"

    printf "\\n\\n===> PUSH IMAGE TO REGISTRY"
    printf "\\n%s\\n" "$HEADER"
    echo "$ docker push \"$REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME\""
    printf "\\n"
    docker push "$REGISTRY_ADDR:$REGISTRY_PORT/$IMAGE_NAME"
}

up_master ()
{
    printf "\\n\\n===> SPIN UP MASTER NODE"
    printf "\\n%s\\n" "$HEADER"
    echo "$ docker-compose up -d master"
    printf "\\n"
    docker-compose up -d master
}


up_workers ()
{
    printf "\\n\\n===> SPIN UP WORKER NODES"
    printf "\\n%s\\n" "$HEADER"
    echo "$ docker-compose up -d worker"
    printf "\\n"
    docker-compose up -d worker 

    printf "\\n"
    printf "\\n%s\\n" "$HEADER"

    NUM_WORKER=$((SIZE - 1))
    echo "$ docker-compose scale worker=$NUM_WORKER"
    printf "\\n"
    docker-compose scale worker=${NUM_WORKER}
}

down_master ()
{
    printf "\\n\\n===> TORN DOWN MASTER NODE"
    printf "\\n%s\\n" "$HEADER"

    echo "$ docker-compose stop master && docker-compose rm -f master"
    printf "\\n"
    docker-compose stop master && docker-compose rm -f master
}

down_workers ()
{
    printf "\\n\\n===> TORN DOWN WORKER NODES"
    printf "\\n%s\\n" "$HEADER"
    echo "$ docker-compose stop worker && docker-compose rm -f worker"
    printf "\\n"
    docker-compose stop worker && docker-compose rm -f worker
}

list ()
{
    printf "\\n\\n===> LIST CONTAINERS"
    printf "\\n%s\\n" "$HEADER"
    echo "$ docker-compose ps"
    printf "\\n"
    docker-compose ps
}


exec_on_mpi_master_container ()
{
    # shellcheck disable=SC2046
    docker exec -it -u mpi $(docker-compose ps | grep 'master'| awk 'NR==1{print $1}') "$@"
}

prompt_ready ()
{
    printf "\\n\\n===> SWARMLAB READY \\n\\n"
}

show_instruction ()
{
    echo '                            ##         .          '
    echo '                      ## ## ##        ==          '
    echo '                   ## ## ## ## ##    ===          '
    echo '               /"""""""""""""""""\___/ ===        '
    echo '          ~~~ {~~ ~~~~ ~~~ ~~~~ ~~~ ~ /  ===- ~~~ '
    echo '               \______ o           __/            '
    echo '                 \    \         __/               '
    echo '                  \____\_______/                  '
    echo '                                                  '
    echo '                 MPICH Swarmlab.io             '
    echo ''
    echo '=============================================================='
    echo ''

    echo "To run MPI programs in an interative shell:"
    echo "  1. Login to master node:"
    echo "     Using Docker through command wrapper:"
    echo "     $ swarmlab-mpi login"
    echo ""
    echo "     Or using SSH with keys through exposed port:"
    echo "     $ ssh -o \"StrictHostKeyChecking no\" -i ssh/id_rsa -p $SSH_PORT mpi@localhost"
    echo '       where [localhost] could be changed to the host IP of master node'
    echo ""
    echo "  2. Execute MPI programs inside master node, for example:"
    echo "     $ mpirun file"
    echo "      *----------------------------------------------------*"
    echo "      | Default hostfile of connected nodes in the swarmlab |"
    echo "      | is automatically updated at /etc/opt/hosts         |"
    echo "      | To obtain hostfile manually: $ get_hosts > hosts   |"
    echo "      * ---------------------------------------------------*"
    echo ""
    echo ""
}



#############################################

while [ "$1" != "" ];
do
    PARAM=$(echo "$1" | awk -F= '{print $1}')
    VALUE=$(echo "$1" | awk -F= '{print $2}')

    case $PARAM in
        help)
            usage
            exit
            ;;
        -i)
            show_instruction
            exit
            ;;

        login)
            COMMAND_LOGIN=1
            ;;

        exec)
            COMMAND_EXEC=1
            shift # the rest is the shell command to run in the node
            SHELL_COMMAND="$*"
            break # end while loop
            ;;

        up)
            COMMAND_UP=1
            ;;

        create)
            COMMAND_CREATE=1
            ;;

        down)
            COMMAND_DOWN=1
            ;;

        reload)
            COMMAND_RELOAD=1
            ;;

        scale)
            COMMAND_SCALE=1
            ;;

        list)
            COMMAND_LIST=1
            ;;

        clean)
            COMMAND_CLEAN=1
            ;;

        size)
            [ "$VALUE" ] && SIZE=$VALUE
            ;;

        *)
            echo "ERROR: unknown parameter \"$PARAM\""
            usage
            exit 1
            ;;
    esac
    shift
done


if [ $COMMAND_UP -eq 1 ]; then
    create_dockerfile
    down_all
    clear_all
    up_registry
    generate_ssh_keys
    build_and_push_image
    up_master
    up_workers

    prompt_ready
    show_instruction

elif [ $COMMAND_CREATE -eq 1 ]; then
    create_project

elif [ $COMMAND_DOWN -eq 1 ]; then
    down_all
    clear_all

elif [ $COMMAND_CLEAN -eq 1 ]; then
    clear_all


elif [ $COMMAND_SCALE -eq 1 ]; then
    create_dockerfile
    down_master
    down_workers
    up_master
    up_workers

    prompt_ready
    show_instruction

elif [ $COMMAND_RELOAD -eq 1 ]; then
    create_dockerfile
    down_master
    down_workers
    build_and_push_image
    up_master
    up_workers

    prompt_ready
    show_instruction

elif [ $COMMAND_LOGIN -eq 1 ]; then
    exec_on_mpi_master_container /bin/sh

elif [ $COMMAND_EXEC -eq 1 ]; then
    create_dockerfile
    exec_on_mpi_master_container ash -c "${SHELL_COMMAND}"

elif [ $COMMAND_LIST -eq 1 ]; then
    list
else
    usage
fi