From 753c90ec1631076bb76a9bfb1eba9c342c25908b Mon Sep 17 00:00:00 2001 From: lefteris Date: Thu, 30 Sep 2021 13:33:36 +0300 Subject: [PATCH] final --- swarmlab-app/src/run/app.js | 564 +++++++++--------------------------- 1 file changed, 139 insertions(+), 425 deletions(-) diff --git a/swarmlab-app/src/run/app.js b/swarmlab-app/src/run/app.js index 2220c52c..1b12bfa2 100644 --- a/swarmlab-app/src/run/app.js +++ b/swarmlab-app/src/run/app.js @@ -98,9 +98,7 @@ const corsOptions = { // Lefos - get length of logs app.get("/length", cors(corsOptions), (req, res) => { console.error("getting length of logs"); - var url = "mongodb://mongo:27017/"; - MongoClient.connect( url, { useNewUrlParser: true, useUnifiedTopology: true }, @@ -114,59 +112,97 @@ app.get("/length", cors(corsOptions), (req, res) => { if (err) throw err; // EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES var obj = JSON.parse(JSON.stringify(result)); - var appout = 0; - var apperror = 0; - var mongol = 0; - var mongolerror = 0; - var redis = 0; - var dumout = 0; - var dumerr = 0; + let containers = []; + var type; + var found = 0; obj.forEach((value) => { - if (value.container_name == "/mongo") { - var tmp = JSON.parse(value.log); - if (tmp.s == "I") mongol++; - // count every informative log as an output log - else if (tmp.s == "W" || tmp.s == "E") mongolerror++; //count every warning and error log - } else if ( - value.container_name == "/readmongo_service" && - value.log.includes("app_name") - ) { - var test = JSON.parse(value.log); - var type = test.type; - if (type == "err") apperror++; - else appout++; - } else if (value.container_name == "/redisserver") { - redis++; - } else if ( - value.container_name == "/dummy_service" && - value.log.includes("app_name") - ) { - var test = JSON.parse(value.log); - var type = test.type; - if (type == "err") dumerr++; - else dumout++; + //new code + if (containers.length == 0) { + type = checkService(); + if (type == "out") { + containers.push({ + name: value.container_name, + lengtho: 1, + lengthe: 0, + }); + } else if (type == "err") { + containers.push({ + name: value.container_name, + lengtho: 0, + lengthe: 1, + }); + } + } else { + containers.forEach((val) => { + if (val.name == value.container_name) { + found = 1; + type = checkService(); + if (type == "out") { + val.lengtho++; + } else if (type == "err") { + val.lengthe++; + } + } + }); + if (found == 0) { + type = checkService(); + if (type == "out") { + containers.push({ + name: value.container_name, + lengtho: 1, + lengthe: 0, + }); + } else if ("error") { + containers.push({ + name: value.container_name, + lengtho: 0, + lengthe: 1, + }); + } + } + found = 0; + } + + // new code end + + function checkService() { + if (value.tag.includes("mongodb")) { + var tmp = JSON.parse(value.log); + if (tmp.s == "I") return "out"; + // count every informative log as an output log + else if (tmp.s == "W" || tmp.s == "E") return "err"; //count every warning and error log + } else if (value.tag.includes("node")) { + if (IsJsonString(value.log)) var test = JSON.parse(value.log); + else return "none"; + + var type = test.type; + if (type == "out") return "out"; + else return "err"; + } else if (value.tag.includes("redis")) { + return "out"; + } } }); var RES = new Object(); - var data = { - mongolength: mongol, - mongolerror: mongolerror, - appout: appout, - apperror: apperror, - redis: redis, - dumout: dumout, - dumerr: dumerr, - }; - RES.data = data; + RES.data = containers; // console.log("Sending Data: " + jsonfinal); - res.json(RES.data); + res.json(containers); db.close(); }); } ); }); +function IsJsonString(str) { + try { + JSON.parse(str); + } catch (e) { + return false; + } + return true; +} + app.get("/services", cors(corsOptions), (req, res) => { console.error("getting length of logs"); @@ -278,6 +314,39 @@ app.get("/test", cors(corsOptions), (req, res) => { ); }); +app.get("/raw", cors(corsOptions), (req, res) => { + console.log("reading from db...."); + + var url = "mongodb://mongo:27017/"; + + MongoClient.connect( + url, + { useNewUrlParser: true, useUnifiedTopology: true }, + function (err, db) { + if (err) throw err; + var dbo = db.db("fluentdb"); + dbo + .collection("test") + .find({}) + .toArray(function (err, result) { + if (err) throw err; + // EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES + var obj = JSON.parse(JSON.stringify(result)); + var jsonfinal = []; + obj.forEach((value) => { + jsonfinal.push(value); + }); + var RES = new Object(); + RES.data = { + all: jsonfinal, + }; + res.json(RES.data); + db.close(); + }); + } + ); +}); + app.get("/test2", cors(corsOptions), (req, res) => { var RES = new Object(); const page = req.query["page"]; @@ -314,31 +383,39 @@ app.get("/test2", cors(corsOptions), (req, res) => { var obj = JSON.parse(JSON.stringify(result)); obj.forEach((value) => { - if (value.log.includes("app_name")) { + if (value.tag.includes("node") && IsJsonString(value.log)) { var tmp = JSON.parse(value.log); var container_name = value.container_name; + var tmptype; + tmp.app_name = container_name; if (tmp.type == "out") { - tmp.type = '
' + tmp.type + "
"; + tmptype = '
' + tmp.type + "
"; } else if (tmp.type == "err") { - tmp.type = '
' + tmp.type + "
"; + tmptype = '
' + tmp.type + "
"; } - + var tmp2 = { + message: tmp.message, + timestamp: value.time, + type: tmptype, + process_id: "-", + app_name: value.container_name, + }; // filtrarisma gia to text poy exei dwsei o xrhsths if ( logtext != "" && tmp.message.includes(logtext) && (container_name == selected || selected == "All") ) { - jsonfinal.push(tmp); + jsonfinal.push(tmp2); } else if (logtext != "" && !tmp.message.includes(logtext)) { } else if ( logtext == "" && (container_name == selected || selected == "All") ) { - jsonfinal.push(tmp); + jsonfinal.push(tmp2); } } else if ( - value.container_name == "/redisserver" && + value.tag.includes("redis") && (value.container_name == selected || selected == "All") ) { var tmplog = value; @@ -349,12 +426,12 @@ app.get("/test2", cors(corsOptions), (req, res) => { timestamp: time, type: "
out
", process_id: "-", - app_name: "redis", + app_name: value.container_name, }; if (logtext == "" || tmp.message.includes(logtext)) jsonfinal.push(tmp); } else if ( - value.container_name == "/mongo" && + value.tag.includes("mongodb") && (value.container_name == selected || selected == "All") ) { var tmplog = JSON.parse(value.log); @@ -367,10 +444,18 @@ app.get("/test2", cors(corsOptions), (req, res) => { timestamp: time, type: "
out
", process_id: "-", - app_name: "Mongo", + app_name: value.container_name, }; if (logtext == "" || tmp.message.includes(logtext)) jsonfinal.push(tmp); + } else { + var tmp = { + message: value.log, + timestamp: value.time, + type: "
" + value.source + "
", + process_id: "-", + app_name: value.container_name, + }; } }); //console.log("Sending Data: " + jsonfinal); @@ -554,6 +639,7 @@ io.on("connection", (s) => { usersession.SOCKET.user = isvalid.user; usersession.SOCKET.scope = isvalid.scope; // space delimeter usersession.SOCKET.token = isvalid.token; + usersession.SOCKET.id = s.id; //console.log("Reply: " + rep); // ----- @@ -873,351 +959,6 @@ async function checkstream(data) { ); } } -// *************************************************** -// rest get -// *************************************************** - -// app.get( -// "/get_log", -// [check("token").isLength({ min: 40 })], -// cors(corsOptions), -// (req, res, next) => { -// (async () => { -// var RES = new Object(); -// RES.token = req.query["token"]; -// RES.start = req.query["start"]; -// RES.end = req.query["end"]; -// RES.swarmlabname = req.query["swarmlabname"]; -// RES.ok = "ok"; -// /* -// * -// * validate -// * -// */ - -// var isvalid = await checkToken(RES.token); -// if (isvalid.action == "ok") { -// console.log("Authserver ok " + RES.token); -// RES.error = "ok"; -// } else { -// console.log("Authserver no " + RES.token); -// RES.error = "no"; -// } -// if (RES.error == "ok") { -// var resdata = await getuserpipelines( -// RES.token, -// isvalid.user, -// RES.swarmlabname -// ); -// var mongourl = -// "mongodb://" + -// CONFIG.mongo.user + -// ":" + -// CONFIG.mongo.password + -// "@ondemand_playground_mongo1:27017,ondemand_playground_mongo2:27017,ondemand_playground_mongo3:27017,ondemand_playground_mongo4:27017,ondemand_playground_mongo5:27017,ondemand_playground_mongo6:27017,ondemand_playground_mongo7:27017/swarmlabplaygroundstats?replicaSet=rs1&authSource=swarmlabplaygroundstats"; -// const OPTS = { -// useNewUrlParser: true, -// useUnifiedTopology: true, -// }; -// MongoClient.connect(mongourl, OPTS, function (err, client) { -// if (err) { -// console.log(err); -// } else { -// const db = client.db("swarmlabplaygroundstats"); -// //usersession.SOCKET.user = isvalid.user -// console.log(JSON.stringify("mongo ----------------connected")); -// console.log("-----test------- " + JSON.stringify(RES)); -// if ( -// typeof RES.start !== "undefined" && -// typeof RES.end !== "undefined" -// ) { -// if (DateTime.fromISO(RES.start).isValid) { -// var datestart = DateTime.fromISO(RES.start); -// var dateend = DateTime.fromISO(RES.end); -// var search_term = { -// $and: [ -// { -// time: { -// $gte: datestart, -// }, -// }, -// { -// time: { -// $lt: dateend, -// }, -// }, -// ], -// }; -// } else { -// RES.ok = "no"; -// } -// } else if (typeof RES.end !== "undefined") { -// var dateend = DateTime.fromISO(RES.end); -// if (DateTime.fromISO(RES.end).isValid) { -// var search_term = { -// $and: [ -// { -// time: { -// $lt: dateend, -// }, -// }, -// ], -// }; -// } else { -// RES.ok = "no"; -// } -// } else if (typeof RES.start !== "undefined") { -// var datestart = DateTime.fromISO(RES.start); -// if (DateTime.fromISO(RES.start).isValid) { -// var search_term = { -// $and: [ -// { -// time: { -// $gte: datestart, -// }, -// }, -// ], -// }; -// } else { -// RES.ok = "no"; -// } -// } -// if (RES.ok == "ok") { -// //var search_term = '{"$gte": new Date("2020-12-01T00:00:00.000Z") , "$lt": new Date("2020-12-11T16:17:36.470Z") }' -// //var search_term = {"time" : {$lte : datenow}} -// var resdataarray = []; -// var resraw = {}; -// var reslab = ""; -// var datestart1 = DateTime.fromISO(RES.start); -// console.log("-----now1------- " + JSON.stringify(search_term)); -// console.log("-----now2------- " + JSON.stringify(datestart1)); -// console.log("-----now3------- " + JSON.stringify(datestart)); - -// db.collection("logs") -// .find(search_term) -// .toArray() -// //db.collection('logs').find({"time" : {$gt : datestart}}).toArray() -// .then((item) => { -// console.log("item " + JSON.stringify(item)); -// for (let i in item) { -// reslab = item[i].tailed_path; -// var segment_array = reslab.split("/"); -// var last_segment = segment_array.pop(); -// var fieldstmp = last_segment.split("-"); -// var nameofswarmlab = fieldstmp[0]; - -// var regexlog = new RegExp(nameofswarmlab); -// for (let ii in resdata.data) { -// if (regexlog.test(resdata.data[ii].res25swarmlabname)) { -// resdataarray.push(item[i]); -// RES.found = item[i]; -// } -// } -// } - -// RES.error_msg = "ok"; -// RES.data = resdataarray; -// //RES.dataserver = resdataarray -// //RES.dataservertmp = resdata -// res.json(RES); -// }) -// .catch((err) => { -// console.error(err); -// RES.error_msg = err; -// res.json(RES); -// }); -// } else { -// // RES.ok -// RES.error_msg = "no date"; -// res.json(RES); -// } -// } // error mongo connect -// }); // mongo connect -// } else { -// // token error -// RES.data = "no"; -// RES.error_msg = "token err"; -// res.json(RES); -// } -// })(); -// } -// ); - -// app.get( -// "/run", -// [ -// //check('access_token').isLength({ min: 40 }), -// //check('llo').isBase64() -// ], -// cors(corsOptions), -// (req, res, next) => { -// (async () => { -// var mongourl = -// "mongodb://" + -// CONFIG.mongo.user + -// ":" + -// CONFIG.mongo.password + -// "@ondemand_playground_mongo1:27017,ondemand_playground_mongo2:27017,ondemand_playground_mongo3:27017,ondemand_playground_mongo4:27017,ondemand_playground_mongo5:27017,ondemand_playground_mongo6:27017,ondemand_playground_mongo7:27017/swarmlabplaygroundstats?replicaSet=rs1&authSource=swarmlabplaygroundstats"; -// const OPTS = { -// useNewUrlParser: true, -// useUnifiedTopology: true, -// }; -// MongoClient.connect(mongourl, OPTS, function (err, client) { -// if (err) { -// console.log(err); -// } else { -// const db = client.db("swarmlabplaygroundstats"); -// //db.collection('log', onCollection); -// console.log(JSON.stringify("mongo connected")); -// var stream = db -// .collection("logs") -// .find( -// {}, -// { -// tailable: true, -// awaitdata: true, -// /* other options */ -// } -// ) -// .stream(); - -// stream.on("data", function (doc) { -// console.log(JSON.stringify(doc)); -// //socket.write(JSON.stringify({'action': 'log','param': doc.log})); -// }); -// } -// }); -// var RES = new Object(); -// RES.code = req.query["filter"]; -// RES.token = req.query["filter"]; -// var isvalid = await checkToken(RES.token); -// if (isvalid.action == "ok") { -// console.log("Authserver ok " + RES.token); -// } else { -// console.log("Authserver no " + RES.token); -// } -// RES.error = false; -// RES.error_msg = "ok"; -// res.json(RES); -// })(); -// } -// ); - -// // *************************************************** -// // rest post -// // *************************************************** - -// app.post( -// "/run", -// [ -// //check('access_token').isLength({ min: 40 }), -// //check('llo').isBase64() -// ], -// cors(corsOptions), -// (req, res, next) => { -// (async () => { -// //console.log(JSON.stringify(req.headers)); -// //console.log(JSON.stringify(req.body)); -// //console.log("mongo "+JSON.stringify(req.body)); -// //console.log("LOG "+JSON.stringify(req.body[0].message)); -// //console.log("PATH "+JSON.stringify(req.body[0].tailed_path)); -// for (var i = 0; i < req.body.length; i++) { -// //var getpath = await onlogfile(req.body[i].tailed_path) - -// var path = req.body[i].tailed_path; - -// console.log("File", path, "has been added"); -// var pathfileval = pathmodule.basename(path); -// var arrfile = pathfileval.toString().split("-"); -// var pathfile = arrfile[0]; -// var indexfind1 = global.pipelines.findIndex( -// (x) => x.pathlogfile == pathfileval -// ); -// console.log( -// "file11111111111111111111111111111111 " + JSON.stringify(pathfileval) -// ); -// if (indexfind1 === -1) { -// (async () => { -// console.log( -// "file2222222222222222222222222222222222222 " + -// JSON.stringify(pathfileval) -// ); -// var token = "d2539e5a7ae1f9f1b0eb2b8f22ca467a86d28407"; // desto -// var resdata = await getpipelines(token, pathfile); -// //resdata.data.pathlogfile = 'test' -// var resob = {}; -// resob.pathlogfile = pathfileval; -// var resobarray = []; -// for (let i in resdata.data) { -// var resob1 = {}; -// resob1.data = resdata.data[i].res25swarmlabname; -// resob1.user25user = resdata.data[i].res25user; -// resob1.res25creator = resdata.data[i].res25creator; -// resob1.res25fileforce = resdata.data[i].res25fileforce; -// resobarray.push(resob1); -// } -// resob.data = resobarray; -// //var indexfind = global.pipelines.findIndex(x => x.res25swarmlabname==resdata.data[0].res25swarmlabname); -// var indexfind = global.pipelines.findIndex( -// (x) => x.pathlogfile == pathfileval -// ); - -// //indexfind === -1 ? global.pipelines.push({resob}) : console.log("object already exists") -// indexfind === -1 -// ? global.pipelines.push(resob) -// : console.log("object already exists " + pathfileval); - -// //console.log('info', JSON.stringify(resdata)); -// //console.log('info------------- ', JSON.stringify(global.pipelines)); -// })(); -// } -// // -// var obj = req.body[i]; - -// //var indexfind = global.pipelines.findIndex(x => x.res25swarmlabname==resdata.data.res25swarmlabname); -// //indexfind === -1 ? global.pipelines.push(resob) : console.log("object already exists") -// var now = new Date(); - -// var reslog = new Object(); -// reslog.log = obj; - -// reslog.date = convertDateToUTC(now); -// console.log(reslog); -// var pathfileval = pathmodule.basename(reslog.log.tailed_path); -// var indexfind = global.pipelines.findIndex( -// (x) => x.pathlogfile == pathfileval -// ); -// //indexfind === -1 ? sendlog(reslog,pathfileval) : console.log("object already exists") -// indexfind === -1 -// ? console.log("object not found") -// : sendlog(reslog, pathfileval); -// console.log("IOT " + JSON.stringify(reslog.log.tailed_path)); -// console.log("IOTindexfind " + JSON.stringify(indexfind)); -// console.log("IOTuser " + JSON.stringify(global.pipelines)); -// // io.in("iot").emit("message", reslog); -// // io.emit("logdata", reslog); -// } -// })(); - -// //io.in("iot").emit("message", RES); - -// console.error("socket POST from client"); -// var RES = new Object(); -// RES.error = false; -// RES.error_msg = "ok"; -// RES.msg = req.body[0].messsage; - -// res.json(RES); -// } -// ); - -// *************************************************** -// socket -// *************************************************** - -//function getSHA256ofJSON(input){ -// return require("crypto").createHash("sha256").update(JSON.stringify(input)).digest("hex"); -//} function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -1403,32 +1144,5 @@ function onCollection(err, collection) { }, 8000); } -/// ARXIKO MONGO CONNECTION /// -// var mongourl = -// "mongodb://" + -// CONFIG.mongo.user + -// ":" + -// CONFIG.mongo.password + -// "@ondemand_playground_mongo1:27017,ondemand_playground_mongo2:27017,ondemand_playground_mongo3:27017,ondemand_playground_mongo4:27017,ondemand_playground_mongo5:27017,ondemand_playground_mongo6:27017,ondemand_playground_mongo7:27017/swarmlabplaygroundstats?replicaSet=rs1&authSource=swarmlabplaygroundstats"; -// const OPTS = { -// useNewUrlParser: true, -// useUnifiedTopology: true, -// }; -// var mongooptions = { -// autoReconnect: true, -// keepAlive: 1, -// connectTimeoutMS: 30000, -// socketTimeoutMS: 0, -// }; - -// MongoClient.connect(mongourl, OPTS, function (err, client) { -// if (err) { -// console.log(err); -// } else { -// const db = client.db("swarmlabplaygroundstats"); -// db.collection("logs", onCollection); -// } -// }); - http.listen(3000, () => console.error("listening on http://localhost:3000/")); console.error("socket.io example");