Add files via upload

3 years ago · e7a0af9519
7 changed files with 628 additions and 0 deletions
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 20.8b1 
+    hooks:
+      - id: black
+        language_version: python3 
+        exclude: stresstest-locusts/
--- a/check_ports.sh
+++ b/check_ports.sh
@ -0,0 +1,23 @@
+#!/bin/bash
+arr=("$@")
+   for i in "${arr[@]}";
+      do
+          echo "$i"
+      done
+isopenport()
+{
+    arr=("$@")
+    for i in "${arr[@]}";
+        do
+            nc -z 127.0.0.1 $i &> /dev/null
+            result1=$?
+            if [  "$result1" != 0 ]; then
+                echo  port $i is free 
+            else
+                echo port $i is used  
+            fi
+        done
+}
+ports=(800 801 81 82 83 84 85 86 88 89 1883 27017 8080 3000 6379)
+
+isopenport "${ports[@]}" 
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,284 @@
+version: '3.8'
+
+services:
+    # base image for building our python apps, used for speeding up the build process
+    web-base-image:
+        build:
+            context: ./docker-python/base
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-base
+
+    # generates random numbers and lists them
+    # uses healthcheck by making a curl request to a GET endpoint
+    web-random:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-standard
+        ports:
+            - "800:5000"
+        entrypoint: python /root/flask-mongodb-example/python/random_demo.py
+        depends_on:
+            - mongo
+            - web-base-image
+        healthcheck:
+            test: curl --silent --show-error --fail "http://localhost:5000/random?lower=0&upper10"
+            interval: 10s
+            start_period: 30s
+
+    # generates random numbers and lists them, it's faster thatn web-random because it's using pypy accelerator
+    web-random-pypy:
+        build:
+            context: ./docker-python-pypy
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-standard-pypy
+        ports:
+            - "801:5000"
+        entrypoint: pypy /root/flask-mongodb-example/python/random_demo.py
+        depends_on:
+            - mongo
+
+    # Create, read, update and delete operations over a user collection
+    web-users:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-standard
+        ports:
+            - "81:5000"
+        entrypoint: python /root/flask-mongodb-example/python/users.py
+        environment:
+            - REDIS_PASSWORD=/run/secrets/redis_password
+            - PYTHONUNBUFFERED=1
+        secrets:
+            - redis_password
+        depends_on:
+            - mongo
+            - web-base-image
+            - redis
+
+    # Uses an MQTT server (Mosquitto) to allow to publish sensor updates over MQTT
+    background-mqtt:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements-mqtt.txt
+        image: backgorund-mqtt-image
+        entrypoint: python /root/flask-mongodb-example/python/mqtt.py
+        environment:
+            - MQTT_USER=/run/secrets/mqtt_user
+            - MQTT_PASSWORD=/run/secrets/mqtt_password
+            - PYTHONUNBUFFERED=1
+        secrets:
+            - mqtt_user
+            - mqtt_password
+        depends_on:
+            - mongo
+            - mqtt
+            - influxdb
+        tty: true
+
+    # Fulltext search engine backed by fulltext MongoDb index
+    web-fulltext-search:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-standard
+        ports:
+            - "82:5000"
+        entrypoint: python /root/flask-mongodb-example/python/fulltext_search.py
+        depends_on:
+            - mongo
+
+    # Geospacial search service that supports adding places, and quering the placing by coordonates and distance
+    web-geolocation-search:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-geolocation-image
+        ports:
+            - "83:5000"
+        entrypoint: python /root/flask-mongodb-example/python/geolocation_search.py
+        depends_on:
+            - mongo
+
+    # Baesian average demo (https://en.wikipedia.org/wiki/Bayesian_average)
+    web-baesian:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-standard
+        ports:
+            - "84:5000"
+        entrypoint: python /root/flask-mongodb-example/python/baesian.py
+        depends_on:
+            - mongo
+
+    # A demo of working with file photo uploads, hash searching and using docker volumes
+    web-photo-process:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements-photo.txt
+        image: web-photo-image
+        ports:
+            - "85:5000"
+        entrypoint: python /root/flask-mongodb-example/python/photo_process.py
+        volumes:
+            - ./container-storage:/root/storage
+        depends_on:
+            - mongo
+
+    # A virtual book library
+    web-book-collection:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements-restplus.txt
+        image: web-bookcollection-image
+        environment:
+             - PYTHONUNBUFFERED=1
+        ports:
+            - "86:5000"
+        entrypoint: python /root/flask-mongodb-example/python/bookcollection.py
+        depends_on:
+            - mongo
+            - web-users
+
+    # Wame functionality as web-users but build with fastapi
+    # runs with gunicorn on two processor cores [-w 2]
+    web-users-fast-api:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements-fastapi.txt
+        image: web-users-fastapi-image
+        ports:
+            - "88:5000" # port 87 is restricted in browsers
+        entrypoint: gunicorn -w 2 -k uvicorn.workers.UvicornH11Worker --bind 0.0.0.0:5000 --log-level debug users-fastapi:app
+        depends_on:
+            - mongo
+
+    # A tic tac toe game written in flask using flask_session. It has a simple UI
+    web-tictactoe:
+        build:
+            context: ./docker-python
+            args:
+                requirements: /root/flask-mongodb-example/python/requirements.txt
+        image: web-standard
+        ports:
+            - "89:5000"
+        entrypoint: python /root/flask-mongodb-example/python/tictactoe.py
+
+    # Database for grafana
+    influxdb:
+        build: docker-influxdb
+        env_file: ./docker-influxdb/configuration.env
+        ports:
+          - '8086:8086'
+        volumes:
+          - influxdb_data:/var/lib/influxdb
+
+    # For showing graphs on sensors
+    grafana:
+        build: docker-grafana
+        env_file: ./docker-grafana/configuration.env
+        links:
+          - influxdb
+        ports:
+          - '3000:3000'
+        volumes:
+          - grafana_data:/var/lib/grafana
+
+    # Genaral purpose database
+    # uses healthcheck for replicasset rs.status().ok
+    mongo:
+        image: mongo:4.4.5-bionic
+        hostname: mongodb
+        ports:
+            - '27017:27017'
+        environment:
+            - MONGO_REPLICA_SET_NAME=dan
+        command:
+            - --storageEngine
+            - wiredTiger
+            - --replSet
+            - myrepl
+        healthcheck:
+            test: test $$(echo "rs.initiate().ok || rs.status().ok" | mongo  --quiet) -eq 1
+            interval: 10s
+            start_period: 30s
+
+    # Redis server, used here for caching
+    # uses healthcheck with redis-cli ping
+    redis:
+        build:
+            context: ./docker-redis
+        ports:
+            - '6379:6379'
+        healthcheck:
+              test: ["CMD", "redis-cli", "ping"]
+              interval: 10s
+              timeout: 3s
+              start_period: 30s
+
+    # Mosquitto MQTT broker
+    mqtt:
+        build:
+          context: ./docker-mosquitto
+        ports:
+            - "1883:1883"
+
+    # An api gateway
+    krakend:
+        image: devopsfaith/krakend
+        volumes:
+            - ${PWD}/krakend.json:/etc/krakend/krakend.json
+        ports:
+            - "8080:8080"
+        depends_on:
+            - web-random
+            - web-users
+
+    # Used to test services
+    web-test:
+        image: alpine
+        depends_on:
+            - web-base-image
+            - web-random
+            - web-random-pypy
+            - web-users
+            - background-mqtt
+            - web-fulltext-search
+            - web-geolocation-search
+            - web-baesian
+            - web-photo-process
+            - web-book-collection
+            - web-users-fast-api
+            - influxdb
+            - grafana
+            - mongo
+            - mqtt
+            - krakend
+            - web-tictactoe
+            - redis
+
+
+volumes:
+    grafana_data: {}
+    influxdb_data: {}
+
+secrets:
+    mqtt_user:
+        file: ./secrets/mqtt_user.txt
+    mqtt_password:
+        file: ./secrets/mqtt_pass.txt
+    redis_password:
+        file: ./secrets/redis_pass.txt
--- a/import.sh
+++ b/import.sh
@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+echo "Runing import script"
+# random_demo
+curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100"
+curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100"
+curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100"
+curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100"
+curl -X PUT "http://localhost:800/random" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "lower=10&upper=100"
+
+# users crud
+curl -X PUT "http://localhost:81/users/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "email=daniel%40gmail.com&name=Daniel"
+curl -X PUT "http://localhost:81/users/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "email=gimy%40gmail.com&name=Gimy"
+curl -X PUT "http://localhost:81/users/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "email=tor%40gmail.com&name=Tor"
+
+# fulltext search
+curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=Who%20has%20many%20apples%3F"
+curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=The%20apple%20tree%20grew%20in%20the%20park"
+curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=Some%20apples%20are%20green%20some%20are%20yellow"
+curl -X PUT "http://localhost:82/fulltext" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "expression=How%20many%20trees%20are%20there%20in%20this%20forest%3F"
+
+# geolocation-search
+curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Bucharest&lat=44.2218653&lng=26.1753655"
+curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Sofia&lat=42.6954108&lng=23.2539076"
+curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Belgrade&lat=44.8152239&lng=20.3525579"
+curl -X POST "http://localhost:83/location" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Minsk&lat=53.8846196&lng=27.5233296"
+
+# baesian
+curl -X POST "http://localhost:84/item/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Dan"
+curl -X POST "http://localhost:84/item/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=John"
+curl -X POST "http://localhost:84/item/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "name=Cicero"
+
+curl -X PUT "http://localhost:84/item/vote/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=6&userid=1"
+curl -X PUT "http://localhost:84/item/vote/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=8&userid=1"
+curl -X PUT "http://localhost:84/item/vote/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=9&userid=1"
+curl -X PUT "http://localhost:84/item/vote/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=8&userid=2"
+curl -X PUT "http://localhost:84/item/vote/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=8&userid=2"
+curl -X PUT "http://localhost:84/item/vote/3" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=4&userid=3"
+curl -X PUT "http://localhost:84/item/vote/1" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=4&userid=3"
+curl -X PUT "http://localhost:84/item/vote/2" -H "accept: application/json" -H "Content-Type: application/x-www-form-urlencoded" -d "mark=9&userid=3"
+
+# bookcollection
+curl -X PUT "http://localhost:86/book/978-0735211292" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"isbn\": \"978-0735211292\", \"name\": \"Atomic Habits: An Easy & Proven Way to Build Good Habits & Break Bad Ones\", \"author\": \"James Clear\", \"publisher\": \"Avery; Illustrated Edition (October 16, 2018)\", \"nr_available\": 5}"
+curl -X PUT "http://localhost:86/book/978-0525538585" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"isbn\": \"978-0525538585\", \"name\": \"Stillness Is the Key\", \"author\": \"Ryan Holiday\", \"publisher\": \"Portfolio (October 1, 2019)\", \"nr_available\": 3}"
+curl -X PUT "http://localhost:86/borrow/1" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"id\": \"string\", \"userid\": 1, \"isbn\": \"978-0735211292\", \"borrow_date\": \"2020-10-13T13:04:37.644Z\", \"max_return_date\": \"2020-11-13T13:04:37.644Z\"}"
--- a/krakend.json
+++ b/krakend.json
@ -0,0 +1,90 @@
+{
+  "version": 2,
+  "extra_config": {
+    "github_com/devopsfaith/krakend-gologging": {
+      "level": "INFO",
+      "prefix": "[KRAKEND]",
+      "syslog": false,
+      "stdout": true,
+      "format": "default"
+    }
+  },
+  "timeout": "3000ms",
+  "cache_ttl": "300s",
+  "output_encoding": "json",
+  "name": "docker-flask-mongodb-example",
+  "endpoints": [
+    {
+      "endpoint": "/random",
+      "method": "GET",
+      "extra_config": {},
+      "output_encoding": "string",
+      "concurrent_calls": 1,
+      "querystring_params": [
+        "lower",
+        "upper"
+      ],
+      "backend": [
+        {
+          "url_pattern": "/random",
+          "encoding": "string",
+          "sd": "static",
+          "extra_config": {},
+          "method": "GET",
+          "host": [
+            "http://web-random:5000"
+          ],
+          "disable_host_sanitize": false
+        }
+      ]
+    },
+    {
+      "endpoint": "/users",
+      "method": "GET",
+      "extra_config": {},
+      "output_encoding": "string",
+      "concurrent_calls": 1,
+      "querystring_params": [
+        "limit",
+        "offset"
+      ],
+      "backend": [
+        {
+          "url_pattern": "/users",
+          "encoding": "string",
+          "sd": "static",
+          "extra_config": {},
+          "method": "GET",
+          "host": [
+            "http://web-users:5000"
+          ],
+          "disable_host_sanitize": false
+        }
+      ]
+    },
+    {
+      "endpoint": "/random",
+      "method": "PUT",
+      "output_encoding": "string",
+      "extra_config": {},
+      "querystring_params": [
+        "lower",
+        "upper"
+      ],
+      "backend": [
+        {
+          "url_pattern": "/random",
+          "encoding": "string",
+          "sd": "static",
+          "method": "PUT",
+          "extra_config": {},
+          "host": [
+            "http://web-random:5000"
+          ],
+          "disable_host_sanitize": false
+        }
+      ]
+    }
+  ],
+  "disable_rest": false
+}
--- a/test.py
+++ b/test.py
@ -0,0 +1,10 @@
+from faker import Faker
+from faker.providers import geo
+from faker.providers import company
+
+
+faker = Faker("en_US")
+faker.add_provider(geo)
+print(faker.longitude())
+print(faker.latitude())
+print(faker.location_on_land())
--- a/test.txt
+++ b/test.txt
@ -0,0 +1,169 @@
+# ---for testing random---
+curl -X GET -i http://localhost:800/random-list
+curl --get -X GET -i http://localhost:800/random -d lower=50 -d upper=100
+curl -X PUT -i http://localhost:800/random -d upper=100 -d lower=10
+
+#---user crud test---
+# adding users
+curl -X POST -i http://localhost:81/users/1 -d email=john@doe.com -d name=John
+curl -X POST -i http://localhost:81/users/2 -d email=steve@rogers.com -d name=Steve
+curl -X POST -i http://localhost:81/users/3 -d email=change@user.com -d name=Change
+
+# change email of user 3
+curl -X PUT -i http://localhost:81/users/3 -d email=newuser@user.com
+
+# check the change
+curl -X GET -i http://localhost:81/users/3
+
+# delete user 3
+curl -X DELETE -i http://localhost:81/users/3
+
+# check if delete works
+curl -X GET -i http://localhost:81/users
+
+
+#---fulltext search---
+curl -X PUT -i http://localhost:82/fulltext -d expression="Who has many apples"
+curl -X PUT -i http://localhost:82/fulltext -d expression="The apple tree grew in the park"
+curl -X PUT -i http://localhost:82/fulltext -d expression="Some apples are green and some are yellow"
+curl -X PUT -i http://localhost:82/fulltext -d expression="How many trees are there in this forest"
+
+curl -X GET -i http://localhost:82/search/apples
+
+#---geo location search---
+curl -X POST -i http://localhost:83/location \
+    -d name=Bucharest \
+    -d lat="26.1496616" \
+    -d lng="44.4205455"
+
+curl -X GET -i http://localhost:83/location/26.1/44.4
+
+curl -X GET -i http://localhost:83/location/26.1/44.4 -d max_distance==50000
+
+#---Bayesian average---
+curl -X POST -i http://localhost:84/item/1 -d name=Hamlet
+curl -X POST -i http://localhost:84/item/2 -d name=Cicero
+curl -X POST -i http://localhost:84/item/3 -d name=Alfred
+
+curl -X PUT -i http://localhost:84/item/vote/1 -d mark=9 -d userid=1
+curl -X PUT -i http://localhost:84/item/vote/2 -d mark=9 -d userid=4
+curl -X PUT -i http://localhost:84/item/vote/3 -d mark=7 -d userid=6
+
+curl -X DELETE -i http://localhost:84/item/3
+
+curl -X GET -i http://localhost:84/item/1
+curl -X GET -i http://localhost:84/items
+
+#---photo process---
+curl -X PUT -F file=@image1.jpeg -i http://localhost:85/photo/1
+curl -X PUT -F file=@image2.jpeg -i http://localhost:85/photo/2 
+
+curl -X GET http://localhost:85/photo/1 -d resize==100 > image1resize.jpeg
+
+curl -X PUT -F file=@image1resize.jpeg -i http://localhost:85/photo/similar 
+
+curl -X DELETE -i http://localhost:85/photo/2
+
+
+#---book collection---
+curl -X PUT -i http://localhost:86/book/978-1607965503 \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{ 
+            "isbn": "978-1607965503", 
+            "name": "Lincoln the Unknown", 
+            "author": "Dale Carnegie", 
+            "publisher": "snowballpublishing", 
+            "nr_available": 5
+        }'
+
+curl -X PUT -i http://localhost:86/book/9780262529624 \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "name": "Intro to Computation and Programming using Python",
+            "isbn": "9780262529624",
+            "author": "John Guttag",
+            "publisher": "MIT Press", 
+            "nr_available": 3
+        }'
+
+
+curl -X GET -i http://localhost:86/book/9780262529624
+
+curl -X GET -i http://localhost:86/book/9780262529624
+
+curl -X GET http://localhost:86/book -d limit=5 -d offset=0
+
+# borrow book
+# will have to create user for this to work
+curl -X PUT -i http://localhost:86/borrow/978-1607965503 \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "id": 1, 
+            "userid": 1, 
+            "isbn": "978-1607965503", 
+            "borrow_date": "2019-12-12T09:32:51.715Z", 
+            "return_date": "2020-02-12T09:32:51.715Z", 
+            "max_return_date": "2020-03-12T09:32:51.715Z"
+        }'
+
+# list a borrowed book
+curl -X GET -i http://localhost:86/borrow/978-1607965503 
+
+curl -X PUT -i http://localhost:86/borrow/return/978-1607965503 \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "id": "978-1607965503",
+            "return_date":"2020-02-12T09:32:51.715Z"
+        }'
+
+curl -X GET -i http://localhost:86/borrow -d limit=5 -d offset=0
+
+
+#---fastapi user CRUD---
+curl -X POST -i http://localhost:88/users/1 \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "userid": 1,
+            "email": "john@doe.com" 
+            "name": "John"
+        }'
+
+curl -X POST -i http://localhost:88/users/3  \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "userid": 3,
+            "email": "change@user.com",
+            "name": "Change"
+        }'
+
+curl -X GET -i http://localhost:88/users/2
+
+curl -X PUT -i http://localhost:88/users/3 \
+    -H "accept: application/json" \
+    -H "Content-Type: application/json" \
+    -d '{
+            "userid": 3, 
+            "email": "user@user.com",
+            "name": "Change"
+        }'
+
+curl -X DELETE -i http://localhost:88/users/1
+curl -X GET -i http://localhost:88/users
+
+
+curl -i -XPOST 'http://localhost:8086/write?db=influx' --data-binary 'humidity value=61'
+
+
+# mqtt service
+# new terminal
+mosquitto_pub -h localhost -u some_user -P some_pass -p 1883 -d -t sensors -m "{\"sensor_id\": \"temperature\", \"sensor_value\": 15.2}"
+
+# new terminal for sub
+mosquitto_sub -h localhost -u some_user -P some_pass -p 1883 -d -t sensors
+