From 279d58a51ea442976abed482dabef4bed8b8c5b4 Mon Sep 17 00:00:00 2001 From: zeus Date: Tue, 23 Nov 2021 02:32:01 +0200 Subject: [PATCH] edit readmongo --- readmongo/swarmlab-app/src/run/app.js | 418 +++++--------------------- readmongo/swarmlab-app/src/run/db.js | 20 ++ 2 files changed, 90 insertions(+), 348 deletions(-) create mode 100644 readmongo/swarmlab-app/src/run/db.js diff --git a/readmongo/swarmlab-app/src/run/app.js b/readmongo/swarmlab-app/src/run/app.js index dda2936..bece0c4 100644 --- a/readmongo/swarmlab-app/src/run/app.js +++ b/readmongo/swarmlab-app/src/run/app.js @@ -9,6 +9,8 @@ const dotenv = require("dotenv"); const check123 = dotenv.config(); const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env; +const fs = require('fs'); + const io = require("socket.io")(http, { // pingTimeout: 30000, // allowUpgrades: false, @@ -51,6 +53,10 @@ pubClient.on("connect", function () { const MongoClient = require("mongodb").MongoClient; const { DateTime } = require("luxon"); +const loadDB = require('./db'); + + + var async = require("async"); const { check, validationResult } = require("express-validator"); const urlExistSync = require("url-exist-sync"); @@ -103,104 +109,26 @@ const corsOptions = { // ------------------------------ // Lefos - get length of logs app.get("/length", cors(corsOptions), (req, res) => { - console.error("getting length of logs"); var url = URL; - MongoClient.connect( - url, - { useNewUrlParser: true, useUnifiedTopology: true }, - function (err, db) { - if (err) throw err; - var dbo = db.db(DATABASE); - dbo - .collection(COLLECTION) - .find({}) - .toArray(function (err, result) { - if (err) throw err; - // EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES - var obj = JSON.parse(JSON.stringify(result)); - let containers = []; - var type; - var found = 0; - obj.forEach((value) => { - //new code - if (containers.length == 0) { - type = checkService(); - if (type == "out") { + var RES = new Object(); + var containers = []; + +(async() => { + var RES = new Object() + var RESL = new Object() + const mydb = await loadDB() + RES.data = await mydb.collection(COLLECTION).find({}).toArray() + var jsonfinal = []; + RES.data.forEach((value) => { containers.push({ - name: value.container_name, + name: value, lengtho: 1, lengthe: 0, }); - } else if (type == "err") { - containers.push({ - name: value.container_name, - lengtho: 0, - lengthe: 1, - }); - } - } else { - containers.forEach((val) => { - if (val.name == value.container_name) { - found = 1; - type = checkService(); - if (type == "out") { - val.lengtho++; - } else if (type == "err") { - val.lengthe++; - } - } - }); - if (found == 0) { - type = checkService(); - if (type == "out") { - containers.push({ - name: value.container_name, - lengtho: 1, - lengthe: 0, - }); - } else if ("error") { - containers.push({ - name: value.container_name, - lengtho: 0, - lengthe: 1, - }); - } - } - found = 0; - } - - // new code end - - function checkService() { - if (value.tag.includes("mongodb")) { - var tmp = JSON.parse(value.log); - if (tmp.s == "I") return "out"; - // count every informative log as an output log - else if (tmp.s == "W" || tmp.s == "E") return "err"; //count every warning and error log - } else if (value.tag.includes("node")) { - if (IsJsonString(value.log)) var test = JSON.parse(value.log); - else { - if (value.source == "stdout") return "out"; - else return "err"; - } - - var type = test.type; - if (type == "out") return "out"; - else return "err"; - } else if (value.tag.includes("redis")) { - return "out"; - } - } - }); - var RES = new Object(); - RES.data = containers; - - // console.log("Sending Data: " + jsonfinal); - res.json(containers); - db.close(); - }); - } - ); + }); + RESL.data = containers; + res.json(RESL) +})() }); // έλεγχος αν η δομή είναι κατάλληλη για μετατροπή σε json @@ -216,74 +144,26 @@ function IsJsonString(str) { // endpoint για την αναγνώριση των υπηρεσιών στο δίκτυο app.get("/services", cors(corsOptions), (req, res) => { console.error("getting length of logs"); - - var url = URL; - - MongoClient.connect( - url, - { useNewUrlParser: true, useUnifiedTopology: true }, - function (err, db) { - if (err) throw err; - var dbo = db.db(DATABASE); - dbo - .collection(COLLECTION) - .find({}) - .toArray(function (err, result) { - if (err) throw err; - // EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES - var obj = JSON.parse(JSON.stringify(result)); - var services = []; - var flag = false; - obj.forEach((value) => { - if (flag == false) { - services.push(value.container_name); - flag = true; - } else { - var tmp = "none"; - var count = 0; - services.forEach((val) => { - var length = services.length; - - var found = false; - if ( - val != value.container_name && - count + 1 < length && - found == false - ) { - count++; - //services.push(value.container_name) - } else if ( - val != value.container_name && - count + 1 == length && - found == false - ) { - //found =true; - tmp = value.container_name; - } else if (val == value.container_name) { - found = true; - } - }); - - if (tmp != "none") { - services.push(tmp); - } - } - }); - var RES = new Object(); - - RES.data = services; - // console.log("Sending Data: " + jsonfinal); - res.json(RES.data); - db.close(); - }); - } - ); + +(async() => { + var RES = new Object() + const mydb = await loadDB() + RES.data = await mydb.collection(COLLECTION).find({}).toArray() + var jsonfinal = []; + RES.data.forEach((value) => { + jsonfinal.push(value.tailed_path); + }); + //var services = Object.keys(jsonfinal); + var uniq = [...new Set(jsonfinal)]; + res.json(uniq) +})() }); // Lefos - mongo test read app.get("/test", cors(corsOptions), (req, res) => { console.log("reading from db...."); + fs.writeFileSync('/tmp/test-sync', 'db', {flag:'a+'}); var url = URL; MongoClient.connect( @@ -303,20 +183,13 @@ app.get("/test", cors(corsOptions), (req, res) => { var mongo = []; var app = []; obj.forEach((value) => { - if (value.container_name == "/redisserver") { - // jsonfinal.push(JSON.parse(value.log)); jsonfinal.push(value); - } else if (value.container_name == "/mongo") { - mongo.push(JSON.parse(value.log)); - } else if (value.log.includes("app_name")) { - app.push(JSON.parse(value.log)); - } }); var RES = new Object(); RES.data = { - redis: jsonfinal, - mongo: mongo, - app: app, + redis: '', + mongo: '', + app: jsonfinal }; res.json(RES.data); db.close(); @@ -329,34 +202,20 @@ app.get("/test", cors(corsOptions), (req, res) => { app.get("/raw", cors(corsOptions), (req, res) => { console.log("reading from db...."); - var url = URL; - - MongoClient.connect( - url, - { useNewUrlParser: true, useUnifiedTopology: true }, - function (err, db) { - if (err) throw err; - var dbo = db.db(DATABASE); - dbo - .collection(COLLECTION) - .find({}) - .toArray(function (err, result) { - if (err) throw err; - // EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES - var obj = JSON.parse(JSON.stringify(result)); - var jsonfinal = []; - obj.forEach((value) => { - jsonfinal.push(value); - }); - var RES = new Object(); - RES.data = { +// fs.writeFileSync('/tmp/test-sync', 'db1', {flag:'a+'}); +(async() => { + var RES = new Object() + const mydb = await loadDB() + RES.data = await mydb.collection(COLLECTION).find({}).toArray() + var jsonfinal = []; + RES.data.forEach((value) => { + jsonfinal.push(value.tailed_path); + }); + RES.src = { all: jsonfinal, }; - res.json(RES.data); - db.close(); - }); - } - ); + res.json(RES.src) +})() }); // βασικό endpoint που επιστρέφει κατάλληλα τα logs στην ευρετηρίαση του web-client @@ -379,162 +238,22 @@ app.get("/test2", cors(corsOptions), (req, res) => { console.error("EXTRA PARAMS: " + logtext); //console.log("TEST LOG"); - var url = URL; +(async() => { + var RES = new Object() + const mydb = await loadDB() + RES.data = await mydb.collection(COLLECTION).find({}).toArray() var jsonfinal = []; - MongoClient.connect( - url, - { useNewUrlParser: true, useUnifiedTopology: true }, - function (err, db) { - if (err) throw err; - var dbo = db.db(DATABASE); - dbo - .collection(COLLECTION) - .find({}) - .toArray(function (err, result) { - if (err) throw err; - // EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES - var obj = JSON.parse(JSON.stringify(result)); - - obj.forEach((value) => { - var test = IsJsonString(value.log); - if (value.tag.includes("node") && IsJsonString(value.log)) { - var tmp = JSON.parse(value.log); - var container_name = value.container_name; - var tmptype; - tmp.app_name = container_name; - if (tmp.type == "out") { - tmptype = '
' + tmp.type + "
"; - } else if (tmp.type == "err") { - tmptype = '
' + tmp.type + "
"; - } - var tmp2 = { - message: tmp.message, - timestamp: value.time, - type: tmptype, - process_id: "-", - app_name: value.container_name, - }; - // filtrarisma gia to text poy exei dwsei o xrhsths - if ( - logtext != "" && - tmp.message.includes(logtext) && - (container_name == selected || selected == "All") - ) { - jsonfinal.push(tmp2); - } else if (logtext != "" && !tmp.message.includes(logtext)) { - } else if ( - logtext == "" && - (container_name == selected || selected == "All") - ) { - jsonfinal.push(tmp2); - } - } else if (value.tag.includes("node") && test == false) { - //console.error("EDW EIMAI!!!!"); - var tmptype; - if (value.source == "stdout") { - tmptype = '
' + "out" + "
"; - } else if (value.source == "stderr") { - tmptype = '
' + "err" + "
"; - } - var tmp2 = { - message: value.log, - timestamp: value.time, - type: tmptype, - process_id: "-", - app_name: value.container_name, - }; - if ( - logtext != "" && - tmp2.message.includes(logtext) && - (value.container_name == selected || selected == "All") - ) { - jsonfinal.push(tmp2); - } else if (logtext != "" && !value.log.includes(logtext)) { - //console.error("lathos sinthiki"); - } else if ( - logtext == "" && - (value.container_name == selected || selected == "All") - ) { - jsonfinal.push(tmp2); - } - } else if ( - value.tag.includes("redis") && - (value.container_name == selected || selected == "All") - ) { - var tmplog = value; - //timestamp - var time = tmplog.time; - var tmp = { - message: tmplog.log, - timestamp: time, - type: "
out
", - process_id: "-", - app_name: value.container_name, - }; - if (logtext == "" || tmp.message.includes(logtext)) - jsonfinal.push(tmp); - } else if ( - value.tag.includes("mongodb") && - (value.container_name == selected || selected == "All") - ) { - var tmplog = JSON.parse(value.log); - var msg2 = tmplog.msg; - - //timestam - var time = value.time; + RES.data.forEach((value) => { var tmp = { - message: msg2, - timestamp: time, - type: "
out
", - process_id: "-", - app_name: value.container_name, - }; - if (logtext == "" || tmp.message.includes(logtext)) - jsonfinal.push(tmp); - } else { - var tmp = { - message: value.log, + message: value.message, timestamp: value.time, - type: "
" + value.source + "
", - process_id: "-", - app_name: value.container_name, + type: "
" + value._id+ "
", + process_id: '', + app_name: value.tailed_path, }; - } - }); - //console.log("Sending Data: " + jsonfinal); - if (sortname == "type") { - jsonfinal.sort(function (a, b) { - if (sortorder == "asc") { - return a.type.localeCompare(b.type); - } else { - return b.type.localeCompare(a.type); - } - }); - } else if (sortname == "message") { - jsonfinal.sort(function (a, b) { - if (sortorder == "asc") { - return a.message.localeCompare(b.message); - } else { - return b.message.localeCompare(a.message); - } - }); - } else if (sortname == "time") { - jsonfinal.sort(function (a, b) { - if (sortorder == "asc") { - return a.timestamp.localeCompare(b.timestamp); - } else { - return b.timestamp.localeCompare(a.timestamp); - } - }); - } else if (sortname == "app_name") { - jsonfinal.sort(function (a, b) { - if (sortorder == "asc") { - return a.app_name.localeCompare(b.app_name); - } else { - return b.app_name.localeCompare(a.app_name); - } - }); - } + jsonfinal.push(tmp); + }); + //Pagination var total = jsonfinal.length; var perpage = per_page; @@ -584,13 +303,10 @@ app.get("/test2", cors(corsOptions), (req, res) => { // console.log("Sending Data: " + jsonfinal); res.json(RES.data); - db.close(); - }); - } - ); + +})() }); -console.log("reading from db...."); // συνάρτηση που χρησιμοποιείται ως callback για την δημιουργία stream με τη βάση και συγκεκριμένο user async function onCollectionNew(err, collection) { @@ -639,6 +355,7 @@ async function onCollectionNew(err, collection) { // sockets και διαχείριση συνδέσεων io.on("connection", (s) => { console.error("socket connection"); + //fs.writeFileSync('/tmp/test-sync', 'Hey there!', {flag:'a+'}); var usersession = new Object(); usersession.SOCKET = {}; @@ -652,10 +369,12 @@ io.on("connection", (s) => { s.on("authenticate", function (data) { const token = data; console.log("TEST LOG INSIDE ATHENTICATE SOCKET: " + token); + //fs.writeFileSync('/tmp/test-sync', token, {flag:'a+'}); (async () => { var isvalid = await checkToken(token); if (isvalid.action == "ok") { console.log("Authserver ok ", s.id + " - " + token); + //fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'}); // pubClient.set(session, resob1string, function(err, res) { // }); usersession.SOCKET.user = isvalid.user; @@ -668,6 +387,7 @@ io.on("connection", (s) => { s.auth = true; } else { console.log("Authserver no ", s.id + " - " + token); + //fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'}); s.auth = false; } })(); @@ -679,6 +399,8 @@ io.on("connection", (s) => { user: data, id: s.id, }; + var xxx = JSON.stringify(binddata); + //fs.writeFileSync('/tmp/test-syncx', xxx, {flag:'a+'}); checkstream(binddata); }); diff --git a/readmongo/swarmlab-app/src/run/db.js b/readmongo/swarmlab-app/src/run/db.js new file mode 100644 index 0000000..a872a6c --- /dev/null +++ b/readmongo/swarmlab-app/src/run/db.js @@ -0,0 +1,20 @@ +const MongoClient = require('mongodb').MongoClient; +const dotenv = require("dotenv"); +const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env; + +let db; + +const loadDB = async () => { + if (db) { + return db; + } + try { + const client = await MongoClient.connect(URL, { useNewUrlParser: true, useUnifiedTopology: true }); + db = client.db(DATABASE); + } catch (err) { + console.error(err); + } + return db; +}; + +module.exports = loadDB;