diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..22ca511 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + language_version: python3 + exclude: stresstest-locusts/ diff --git a/check_ports.sh b/check_ports.sh new file mode 100644 index 0000000..afa2da9 --- /dev/null +++ b/check_ports.sh @@ -0,0 +1,23 @@ +#!/bin/bash +arr=("$@") + for i in "${arr[@]}"; + do + echo "$i" + done +isopenport() +{ + arr=("$@") + for i in "${arr[@]}"; + do + nc -z 127.0.0.1 $i &> /dev/null + result1=$? + if [ "$result1" != 0 ]; then + echo port $i is free + else + echo port $i is used + fi + done +} +ports=(800 801 81 82 83 84 85 86 88 89 1883 27017 8080 3000 6379) + +isopenport "${ports[@]}" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ab85fa4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,284 @@ +version: '3.8' + +services: + # base image for building our python apps, used for speeding up the build process + web-base-image: + build: + context: ./docker-python/base + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-base + + # generates random numbers and lists them + # uses healthcheck by making a curl request to a GET endpoint + web-random: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-standard + ports: + - "800:5000" + entrypoint: python /root/flask-mongodb-example/python/random_demo.py + depends_on: + - mongo + - web-base-image + healthcheck: + test: curl --silent --show-error --fail "http://localhost:5000/random?lower=0&upper10" + interval: 10s + start_period: 30s + + # generates random numbers and lists them, it's faster thatn web-random because it's using pypy accelerator + web-random-pypy: + build: + context: ./docker-python-pypy + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-standard-pypy + ports: + - "801:5000" + entrypoint: pypy /root/flask-mongodb-example/python/random_demo.py + depends_on: + - mongo + + # Create, read, update and delete operations over a user collection + web-users: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-standard + ports: + - "81:5000" + entrypoint: python /root/flask-mongodb-example/python/users.py + environment: + - REDIS_PASSWORD=/run/secrets/redis_password + - PYTHONUNBUFFERED=1 + secrets: + - redis_password + depends_on: + - mongo + - web-base-image + - redis + + # Uses an MQTT server (Mosquitto) to allow to publish sensor updates over MQTT + background-mqtt: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements-mqtt.txt + image: backgorund-mqtt-image + entrypoint: python /root/flask-mongodb-example/python/mqtt.py + environment: + - MQTT_USER=/run/secrets/mqtt_user + - MQTT_PASSWORD=/run/secrets/mqtt_password + - PYTHONUNBUFFERED=1 + secrets: + - mqtt_user + - mqtt_password + depends_on: + - mongo + - mqtt + - influxdb + tty: true + + # Fulltext search engine backed by fulltext MongoDb index + web-fulltext-search: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-standard + ports: + - "82:5000" + entrypoint: python /root/flask-mongodb-example/python/fulltext_search.py + depends_on: + - mongo + + # Geospacial search service that supports adding places, and quering the placing by coordonates and distance + web-geolocation-search: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-geolocation-image + ports: + - "83:5000" + entrypoint: python /root/flask-mongodb-example/python/geolocation_search.py + depends_on: + - mongo + + # Baesian average demo (https://en.wikipedia.org/wiki/Bayesian_average) + web-baesian: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-standard + ports: + - "84:5000" + entrypoint: python /root/flask-mongodb-example/python/baesian.py + depends_on: + - mongo + + # A demo of working with file photo uploads, hash searching and using docker volumes + web-photo-process: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements-photo.txt + image: web-photo-image + ports: + - "85:5000" + entrypoint: python /root/flask-mongodb-example/python/photo_process.py + volumes: + - ./container-storage:/root/storage + depends_on: + - mongo + + # A virtual book library + web-book-collection: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements-restplus.txt + image: web-bookcollection-image + environment: + - PYTHONUNBUFFERED=1 + ports: + - "86:5000" + entrypoint: python /root/flask-mongodb-example/python/bookcollection.py + depends_on: + - mongo + - web-users + + # Wame functionality as web-users but build with fastapi + # runs with gunicorn on two processor cores [-w 2] + web-users-fast-api: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements-fastapi.txt + image: web-users-fastapi-image + ports: + - "88:5000" # port 87 is restricted in browsers + entrypoint: gunicorn -w 2 -k uvicorn.workers.UvicornH11Worker --bind 0.0.0.0:5000 --log-level debug users-fastapi:app + depends_on: + - mongo + + # A tic tac toe game written in flask using flask_session. It has a simple UI + web-tictactoe: + build: + context: ./docker-python + args: + requirements: /root/flask-mongodb-example/python/requirements.txt + image: web-standard + ports: + - "89:5000" + entrypoint: python /root/flask-mongodb-example/python/tictactoe.py + + # Database for grafana + influxdb: + build: docker-influxdb + env_file: ./docker-influxdb/configuration.env + ports: + - '8086:8086' + volumes: + - influxdb_data:/var/lib/influxdb + + # For showing graphs on sensors + grafana: + build: docker-grafana + env_file: ./docker-grafana/configuration.env + links: + - influxdb + ports: + - '3000:3000' + volumes: + - grafana_data:/var/lib/grafana + + # Genaral purpose database + # uses healthcheck for replicasset rs.status().ok + mongo: + image: mongo:4.4.5-bionic + hostname: mongodb + ports: + - '27017:27017' + environment: + - MONGO_REPLICA_SET_NAME=dan + command: + - --storageEngine + - wiredTiger + - --replSet + - myrepl + healthcheck: + test: test $$(echo "rs.initiate().ok || rs.status().ok" | mongo --quiet) -eq 1 + interval: 10s + start_period: 30s + + # Redis server, used here for caching + # uses healthcheck with redis-cli ping + redis: + build: + context: ./docker-redis + ports: + - '6379:6379' + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + start_period: 30s + + # Mosquitto MQTT broker + mqtt: + build: + context: ./docker-mosquitto + ports: + - "1883:1883" + + # An api gateway + krakend: + image: devopsfaith/krakend + volumes: + - ${PWD}/krakend.json:/etc/krakend/krakend.json + ports: + - "8080:8080" + depends_on: + - web-random + - web-users + + # Used to test services + web-test: + image: alpine + depends_on: + - web-base-image + - web-random + - web-random-pypy + - web-users + - background-mqtt + - web-fulltext-search + - web-geolocation-search + - web-baesian + - web-photo-process + - web-book-collection + - web-users-fast-api + - influxdb + - grafana + - mongo + - mqtt + - krakend + - web-tictactoe + - redis + + +volumes: + grafana_data: {} + influxdb_data: {} + +secrets: + mqtt_user: + file: ./secrets/mqtt_user.txt + mqtt_password: + file: ./secrets/mqtt_pass.txt + redis_password: + file: ./secrets/redis_pass.txt diff --git a/import.sh b/import.sh new file mode 100644 index 0000000..eec01dd --- /dev/null +++ b/import.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +echo "Runing import script" +# random_demo +curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100" +curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100" +curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100" +curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100" +curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100" + +# users crud +curl -X PUT "http://localhost:81/users/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "email=daniel%40gmail.com&name=Daniel" +curl -X PUT "http://localhost:81/users/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "email=gimy%40gmail.com&name=Gimy" +curl -X PUT "http://localhost:81/users/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "email=tor%40gmail.com&name=Tor" + +# fulltext search +curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=Who%20has%20many%20apples%3F" +curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=The%20apple%20tree%20grew%20in%20the%20park" +curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=Some%20apples%20are%20green%20some%20are%20yellow" +curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=How%20many%20trees%20are%20there%20in%20this%20forest%3F" + +# geolocation-search +curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Bucharest&lat=44.2218653&lng=26.1753655" +curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Sofia&lat=42.6954108&lng=23.2539076" +curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Belgrade&lat=44.8152239&lng=20.3525579" +curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Minsk&lat=53.8846196&lng=27.5233296" + +# baesian +curl -X POST "http://localhost:84/item/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Dan" +curl -X POST "http://localhost:84/item/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=John" +curl -X POST "http://localhost:84/item/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Cicero" + +curl -X PUT "http://localhost:84/item/vote/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=6&userid=1" +curl -X PUT "http://localhost:84/item/vote/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=8&userid=1" +curl -X PUT "http://localhost:84/item/vote/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=9&userid=1" +curl -X PUT "http://localhost:84/item/vote/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=8&userid=2" +curl -X PUT "http://localhost:84/item/vote/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=8&userid=2" +curl -X PUT "http://localhost:84/item/vote/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=4&userid=3" +curl -X PUT "http://localhost:84/item/vote/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=4&userid=3" +curl -X PUT "http://localhost:84/item/vote/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=9&userid=3" + +# bookcollection +curl -X PUT "http://localhost:86/book/978-0735211292" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"isbn\": \"978-0735211292\", \"name\": \"Atomic Habits: An Easy & Proven Way to Build Good Habits & Break Bad Ones\", \"author\": \"James Clear\", \"publisher\": \"Avery; Illustrated Edition (October 16, 2018)\", \"nr_available\": 5}" +curl -X PUT "http://localhost:86/book/978-0525538585" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"isbn\": \"978-0525538585\", \"name\": \"Stillness Is the Key\", \"author\": \"Ryan Holiday\", \"publisher\": \"Portfolio (October 1, 2019)\", \"nr_available\": 3}" +curl -X PUT "http://localhost:86/borrow/1" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"id\": \"string\", \"userid\": 1, \"isbn\": \"978-0735211292\", \"borrow_date\": \"2020-10-13T13:04:37.644Z\", \"max_return_date\": \"2020-11-13T13:04:37.644Z\"}" \ No newline at end of file diff --git a/krakend.json b/krakend.json new file mode 100644 index 0000000..ac2fba1 --- /dev/null +++ b/krakend.json @@ -0,0 +1,90 @@ +{ + "version": 2, + "extra_config": { + "github_com/devopsfaith/krakend-gologging": { + "level": "INFO", + "prefix": "[KRAKEND]", + "syslog": false, + "stdout": true, + "format": "default" + } + }, + "timeout": "3000ms", + "cache_ttl": "300s", + "output_encoding": "json", + "name": "docker-flask-mongodb-example", + "endpoints": [ + { + "endpoint": "/random", + "method": "GET", + "extra_config": {}, + "output_encoding": "string", + "concurrent_calls": 1, + "querystring_params": [ + "lower", + "upper" + ], + "backend": [ + { + "url_pattern": "/random", + "encoding": "string", + "sd": "static", + "extra_config": {}, + "method": "GET", + "host": [ + "http://web-random:5000" + ], + "disable_host_sanitize": false + } + ] + }, + { + "endpoint": "/users", + "method": "GET", + "extra_config": {}, + "output_encoding": "string", + "concurrent_calls": 1, + "querystring_params": [ + "limit", + "offset" + ], + "backend": [ + { + "url_pattern": "/users", + "encoding": "string", + "sd": "static", + "extra_config": {}, + "method": "GET", + "host": [ + "http://web-users:5000" + ], + "disable_host_sanitize": false + } + ] + }, + { + "endpoint": "/random", + "method": "PUT", + "output_encoding": "string", + "extra_config": {}, + "querystring_params": [ + "lower", + "upper" + ], + "backend": [ + { + "url_pattern": "/random", + "encoding": "string", + "sd": "static", + "method": "PUT", + "extra_config": {}, + "host": [ + "http://web-random:5000" + ], + "disable_host_sanitize": false + } + ] + } + ], + "disable_rest": false +} \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..6ede562 --- /dev/null +++ b/test.py @@ -0,0 +1,10 @@ +from faker import Faker +from faker.providers import geo +from faker.providers import company + + +faker = Faker("en_US") +faker.add_provider(geo) +print(faker.longitude()) +print(faker.latitude()) +print(faker.location_on_land()) diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..05243ac --- /dev/null +++ b/test.txt @@ -0,0 +1,169 @@ +# ---for testing random--- +curl -X GET -i http://localhost:800/random-list +curl --get -X GET -i http://localhost:800/random -d lower=50 -d upper=100 +curl -X PUT -i http://localhost:800/random -d upper=100 -d lower=10 + +#---user crud test--- +# adding users +curl -X POST -i http://localhost:81/users/1 -d email=john@doe.com -d name=John +curl -X POST -i http://localhost:81/users/2 -d email=steve@rogers.com -d name=Steve +curl -X POST -i http://localhost:81/users/3 -d email=change@user.com -d name=Change + +# change email of user 3 +curl -X PUT -i http://localhost:81/users/3 -d email=newuser@user.com + +# check the change +curl -X GET -i http://localhost:81/users/3 + +# delete user 3 +curl -X DELETE -i http://localhost:81/users/3 + +# check if delete works +curl -X GET -i http://localhost:81/users + + +#---fulltext search--- +curl -X PUT -i http://localhost:82/fulltext -d expression="Who has many apples" +curl -X PUT -i http://localhost:82/fulltext -d expression="The apple tree grew in the park" +curl -X PUT -i http://localhost:82/fulltext -d expression="Some apples are green and some are yellow" +curl -X PUT -i http://localhost:82/fulltext -d expression="How many trees are there in this forest" + +curl -X GET -i http://localhost:82/search/apples + +#---geo location search--- +curl -X POST -i http://localhost:83/location \ + -d name=Bucharest \ + -d lat="26.1496616" \ + -d lng="44.4205455" + +curl -X GET -i http://localhost:83/location/26.1/44.4 + +curl -X GET -i http://localhost:83/location/26.1/44.4 -d max_distance==50000 + +#---Bayesian average--- +curl -X POST -i http://localhost:84/item/1 -d name=Hamlet +curl -X POST -i http://localhost:84/item/2 -d name=Cicero +curl -X POST -i http://localhost:84/item/3 -d name=Alfred + +curl -X PUT -i http://localhost:84/item/vote/1 -d mark=9 -d userid=1 +curl -X PUT -i http://localhost:84/item/vote/2 -d mark=9 -d userid=4 +curl -X PUT -i http://localhost:84/item/vote/3 -d mark=7 -d userid=6 + +curl -X DELETE -i http://localhost:84/item/3 + +curl -X GET -i http://localhost:84/item/1 +curl -X GET -i http://localhost:84/items + +#---photo process--- +curl -X PUT -F file=@image1.jpeg -i http://localhost:85/photo/1 +curl -X PUT -F file=@image2.jpeg -i http://localhost:85/photo/2 + +curl -X GET http://localhost:85/photo/1 -d resize==100 > image1resize.jpeg + +curl -X PUT -F file=@image1resize.jpeg -i http://localhost:85/photo/similar + +curl -X DELETE -i http://localhost:85/photo/2 + + +#---book collection--- +curl -X PUT -i http://localhost:86/book/978-1607965503 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "isbn": "978-1607965503", + "name": "Lincoln the Unknown", + "author": "Dale Carnegie", + "publisher": "snowballpublishing", + "nr_available": 5 + }' + +curl -X PUT -i http://localhost:86/book/9780262529624 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Intro to Computation and Programming using Python", + "isbn": "9780262529624", + "author": "John Guttag", + "publisher": "MIT Press", + "nr_available": 3 + }' + + +curl -X GET -i http://localhost:86/book/9780262529624 + +curl -X GET -i http://localhost:86/book/9780262529624 + +curl -X GET http://localhost:86/book -d limit=5 -d offset=0 + +# borrow book +# will have to create user for this to work +curl -X PUT -i http://localhost:86/borrow/978-1607965503 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "id": 1, + "userid": 1, + "isbn": "978-1607965503", + "borrow_date": "2019-12-12T09:32:51.715Z", + "return_date": "2020-02-12T09:32:51.715Z", + "max_return_date": "2020-03-12T09:32:51.715Z" + }' + +# list a borrowed book +curl -X GET -i http://localhost:86/borrow/978-1607965503 + +curl -X PUT -i http://localhost:86/borrow/return/978-1607965503 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "id": "978-1607965503", + "return_date":"2020-02-12T09:32:51.715Z" + }' + +curl -X GET -i http://localhost:86/borrow -d limit=5 -d offset=0 + + +#---fastapi user CRUD--- +curl -X POST -i http://localhost:88/users/1 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "userid": 1, + "email": "john@doe.com" + "name": "John" + }' + +curl -X POST -i http://localhost:88/users/3 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "userid": 3, + "email": "change@user.com", + "name": "Change" + }' + +curl -X GET -i http://localhost:88/users/2 + +curl -X PUT -i http://localhost:88/users/3 \ + -H "accept: application/json" \ + -H "Content-Type: application/json" \ + -d '{ + "userid": 3, + "email": "user@user.com", + "name": "Change" + }' + +curl -X DELETE -i http://localhost:88/users/1 +curl -X GET -i http://localhost:88/users + + +curl -i -XPOST 'http://localhost:8086/write?db=influx' --data-binary 'humidity value=61' + + +# mqtt service +# new terminal +mosquitto_pub -h localhost -u some_user -P some_pass -p 1883 -d -t sensors -m "{\"sensor_id\": \"temperature\", \"sensor_value\": 15.2}" + +# new terminal for sub +mosquitto_sub -h localhost -u some_user -P some_pass -p 1883 -d -t sensors +