swarmlab
/
poc-datacollector
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

edit readmongo

master
zeus 3 years ago
parent
08a015a922
commit
279d58a51e
2 changed files with 90 additions and 348 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 406
      readmongo/swarmlab-app/src/run/app.js
  2. 20
      readmongo/swarmlab-app/src/run/db.js

406
readmongo/swarmlab-app/src/run/app.js
View File

@ -9,6 +9,8 @@ const dotenv = require("dotenv");
const check123 = dotenv.config(); const check123 = dotenv.config();
const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env; const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env;
const fs = require('fs');
const io = require("socket.io")(http, { const io = require("socket.io")(http, {
// pingTimeout: 30000, // pingTimeout: 30000,
// allowUpgrades: false, // allowUpgrades: false,
@ -51,6 +53,10 @@ pubClient.on("connect", function () {
const MongoClient = require("mongodb").MongoClient; const MongoClient = require("mongodb").MongoClient;
const { DateTime } = require("luxon"); const { DateTime } = require("luxon");
const loadDB = require('./db');
var async = require("async"); var async = require("async");
const { check, validationResult } = require("express-validator"); const { check, validationResult } = require("express-validator");
const urlExistSync = require("url-exist-sync"); const urlExistSync = require("url-exist-sync");
@ -103,104 +109,26 @@ const corsOptions = {
// ------------------------------ // ------------------------------
// Lefos - get length of logs // Lefos - get length of logs
app.get("/length", cors(corsOptions), (req, res) => { app.get("/length", cors(corsOptions), (req, res) => {
console.error("getting length of logs");
var url = URL; var url = URL;
MongoClient.connect( var RES = new Object();
url, var containers = [];
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) { (async() => {
if (err) throw err; var RES = new Object()
var dbo = db.db(DATABASE); var RESL = new Object()
dbo const mydb = await loadDB()
.collection(COLLECTION) RES.data = await mydb.collection(COLLECTION).find({}).toArray()
.find({}) var jsonfinal = [];
.toArray(function (err, result) { RES.data.forEach((value) => {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
let containers = [];
var type;
var found = 0;
obj.forEach((value) => {
//new code
if (containers.length == 0) {
type = checkService();
if (type == "out") {
containers.push({
name: value.container_name,
lengtho: 1,
lengthe: 0,
});
} else if (type == "err") {
containers.push({
name: value.container_name,
lengtho: 0,
lengthe: 1,
});
}
} else {
containers.forEach((val) => {
if (val.name == value.container_name) {
found = 1;
type = checkService();
if (type == "out") {
val.lengtho++;
} else if (type == "err") {
val.lengthe++;
}
}
});
if (found == 0) {
type = checkService();
if (type == "out") {
containers.push({ containers.push({
name: value.container_name, name: value,
lengtho: 1, lengtho: 1,
lengthe: 0, lengthe: 0,
}); });
} else if ("error") {
containers.push({
name: value.container_name,
lengtho: 0,
lengthe: 1,
});
}
}
found = 0;
}
// new code end
function checkService() {
if (value.tag.includes("mongodb")) {
var tmp = JSON.parse(value.log);
if (tmp.s == "I") return "out";
// count every informative log as an output log
else if (tmp.s == "W" || tmp.s == "E") return "err"; //count every warning and error log
} else if (value.tag.includes("node")) {
if (IsJsonString(value.log)) var test = JSON.parse(value.log);
else {
if (value.source == "stdout") return "out";
else return "err";
}
var type = test.type;
if (type == "out") return "out";
else return "err";
} else if (value.tag.includes("redis")) {
return "out";
}
}
});
var RES = new Object();
RES.data = containers;
// console.log("Sending Data: " + jsonfinal);
res.json(containers);
db.close();
}); });
} RESL.data = containers;
); res.json(RESL)
})()
}); });
// έλεγχος αν η δομή είναι κατάλληλη για μετατροπή σε json // έλεγχος αν η δομή είναι κατάλληλη για μετατροπή σε json
@ -217,73 +145,25 @@ function IsJsonString(str) {
app.get("/services", cors(corsOptions), (req, res) => { app.get("/services", cors(corsOptions), (req, res) => {
console.error("getting length of logs"); console.error("getting length of logs");
var url = URL; (async() => {
var RES = new Object()
MongoClient.connect( const mydb = await loadDB()
url, RES.data = await mydb.collection(COLLECTION).find({}).toArray()
{ useNewUrlParser: true, useUnifiedTopology: true }, var jsonfinal = [];
function (err, db) { RES.data.forEach((value) => {
if (err) throw err; jsonfinal.push(value.tailed_path);
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
var services = [];
var flag = false;
obj.forEach((value) => {
if (flag == false) {
services.push(value.container_name);
flag = true;
} else {
var tmp = "none";
var count = 0;
services.forEach((val) => {
var length = services.length;
var found = false;
if (
val != value.container_name &&
count + 1 < length &&
found == false
) {
count++;
//services.push(value.container_name)
} else if (
val != value.container_name &&
count + 1 == length &&
found == false
) {
//found =true;
tmp = value.container_name;
} else if (val == value.container_name) {
found = true;
}
});
if (tmp != "none") {
services.push(tmp);
}
}
});
var RES = new Object();
RES.data = services;
// console.log("Sending Data: " + jsonfinal);
res.json(RES.data);
db.close();
}); });
} //var services = Object.keys(jsonfinal);
); var uniq = [...new Set(jsonfinal)];
res.json(uniq)
})()
}); });
// Lefos - mongo test read // Lefos - mongo test read
app.get("/test", cors(corsOptions), (req, res) => { app.get("/test", cors(corsOptions), (req, res) => {
console.log("reading from db...."); console.log("reading from db....");
fs.writeFileSync('/tmp/test-sync', 'db', {flag:'a+'});
var url = URL; var url = URL;
MongoClient.connect( MongoClient.connect(
@ -303,20 +183,13 @@ app.get("/test", cors(corsOptions), (req, res) => {
var mongo = []; var mongo = [];
var app = []; var app = [];
obj.forEach((value) => { obj.forEach((value) => {
if (value.container_name == "/redisserver") {
// jsonfinal.push(JSON.parse(value.log));
jsonfinal.push(value); jsonfinal.push(value);
} else if (value.container_name == "/mongo") {
mongo.push(JSON.parse(value.log));
} else if (value.log.includes("app_name")) {
app.push(JSON.parse(value.log));
}
}); });
var RES = new Object(); var RES = new Object();
RES.data = { RES.data = {
redis: jsonfinal, redis: '',
mongo: mongo, mongo: '',
app: app, app: jsonfinal
}; };
res.json(RES.data); res.json(RES.data);
db.close(); db.close();
@ -329,34 +202,20 @@ app.get("/test", cors(corsOptions), (req, res) => {
app.get("/raw", cors(corsOptions), (req, res) => { app.get("/raw", cors(corsOptions), (req, res) => {
console.log("reading from db...."); console.log("reading from db....");
var url = URL; // fs.writeFileSync('/tmp/test-sync', 'db1', {flag:'a+'});
(async() => {
MongoClient.connect( var RES = new Object()
url, const mydb = await loadDB()
{ useNewUrlParser: true, useUnifiedTopology: true }, RES.data = await mydb.collection(COLLECTION).find({}).toArray()
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
var jsonfinal = []; var jsonfinal = [];
obj.forEach((value) => { RES.data.forEach((value) => {
jsonfinal.push(value); jsonfinal.push(value.tailed_path);
}); });
var RES = new Object(); RES.src = {
RES.data = {
all: jsonfinal, all: jsonfinal,
}; };
res.json(RES.data); res.json(RES.src)
db.close(); })()
});
}
);
}); });
// βασικό endpoint που επιστρέφει κατάλληλα τα logs στην ευρετηρίαση του web-client // βασικό endpoint που επιστρέφει κατάλληλα τα logs στην ευρετηρίαση του web-client
@ -379,162 +238,22 @@ app.get("/test2", cors(corsOptions), (req, res) => {
console.error("EXTRA PARAMS: " + logtext); console.error("EXTRA PARAMS: " + logtext);
//console.log("TEST LOG"); //console.log("TEST LOG");
var url = URL; (async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = []; var jsonfinal = [];
MongoClient.connect( RES.data.forEach((value) => {
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
obj.forEach((value) => {
var test = IsJsonString(value.log);
if (value.tag.includes("node") && IsJsonString(value.log)) {
var tmp = JSON.parse(value.log);
var container_name = value.container_name;
var tmptype;
tmp.app_name = container_name;
if (tmp.type == "out") {
tmptype = '<div class="outtype" >' + tmp.type + "</div>";
} else if (tmp.type == "err") {
tmptype = '<div class="errtype" >' + tmp.type + "</div>";
}
var tmp2 = {
message: tmp.message,
timestamp: value.time,
type: tmptype,
process_id: "-",
app_name: value.container_name,
};
// filtrarisma gia to text poy exei dwsei o xrhsths
if (
logtext != "" &&
tmp.message.includes(logtext) &&
(container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
} else if (logtext != "" && !tmp.message.includes(logtext)) {
} else if (
logtext == "" &&
(container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
}
} else if (value.tag.includes("node") && test == false) {
//console.error("EDW EIMAI!!!!");
var tmptype;
if (value.source == "stdout") {
tmptype = '<div class="outtype" >' + "out" + "</div>";
} else if (value.source == "stderr") {
tmptype = '<div class="errtype" >' + "err" + "</div>";
}
var tmp2 = {
message: value.log,
timestamp: value.time,
type: tmptype,
process_id: "-",
app_name: value.container_name,
};
if (
logtext != "" &&
tmp2.message.includes(logtext) &&
(value.container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
} else if (logtext != "" && !value.log.includes(logtext)) {
//console.error("lathos sinthiki");
} else if (
logtext == "" &&
(value.container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
}
} else if (
value.tag.includes("redis") &&
(value.container_name == selected || selected == "All")
) {
var tmplog = value;
//timestamp
var time = tmplog.time;
var tmp = { var tmp = {
message: tmplog.log, message: value.message,
timestamp: time,
type: "<div class= 'outtype'>out</div>",
process_id: "-",
app_name: value.container_name,
};
if (logtext == "" || tmp.message.includes(logtext))
jsonfinal.push(tmp);
} else if (
value.tag.includes("mongodb") &&
(value.container_name == selected || selected == "All")
) {
var tmplog = JSON.parse(value.log);
var msg2 = tmplog.msg;
//timestam
var time = value.time;
var tmp = {
message: msg2,
timestamp: time,
type: "<div class= 'outtype'>out</div>",
process_id: "-",
app_name: value.container_name,
};
if (logtext == "" || tmp.message.includes(logtext))
jsonfinal.push(tmp);
} else {
var tmp = {
message: value.log,
timestamp: value.time, timestamp: value.time,
type: "<div class= 'outtype'>" + value.source + "</div>", type: "<div class= 'outtype'>" + value._id+ "</div>",
process_id: "-", process_id: '',
app_name: value.container_name, app_name: value.tailed_path,
}; };
} jsonfinal.push(tmp);
});
//console.log("Sending Data: " + jsonfinal);
if (sortname == "type") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.type.localeCompare(b.type);
} else {
return b.type.localeCompare(a.type);
}
});
} else if (sortname == "message") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.message.localeCompare(b.message);
} else {
return b.message.localeCompare(a.message);
}
});
} else if (sortname == "time") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.timestamp.localeCompare(b.timestamp);
} else {
return b.timestamp.localeCompare(a.timestamp);
}
});
} else if (sortname == "app_name") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.app_name.localeCompare(b.app_name);
} else {
return b.app_name.localeCompare(a.app_name);
}
}); });
}
//Pagination //Pagination
var total = jsonfinal.length; var total = jsonfinal.length;
var perpage = per_page; var perpage = per_page;
@ -584,13 +303,10 @@ app.get("/test2", cors(corsOptions), (req, res) => {
// console.log("Sending Data: " + jsonfinal); // console.log("Sending Data: " + jsonfinal);
res.json(RES.data); res.json(RES.data);
db.close();
}); })()
}
);
}); });
console.log("reading from db....");
// συνάρτηση που χρησιμοποιείται ως callback για την δημιουργία stream με τη βάση και συγκεκριμένο user // συνάρτηση που χρησιμοποιείται ως callback για την δημιουργία stream με τη βάση και συγκεκριμένο user
async function onCollectionNew(err, collection) { async function onCollectionNew(err, collection) {
@ -639,6 +355,7 @@ async function onCollectionNew(err, collection) {
// sockets και διαχείριση συνδέσεων // sockets και διαχείριση συνδέσεων
io.on("connection", (s) => { io.on("connection", (s) => {
console.error("socket connection"); console.error("socket connection");
//fs.writeFileSync('/tmp/test-sync', 'Hey there!', {flag:'a+'});
var usersession = new Object(); var usersession = new Object();
usersession.SOCKET = {}; usersession.SOCKET = {};
@ -652,10 +369,12 @@ io.on("connection", (s) => {
s.on("authenticate", function (data) { s.on("authenticate", function (data) {
const token = data; const token = data;
console.log("TEST LOG INSIDE ATHENTICATE SOCKET: " + token); console.log("TEST LOG INSIDE ATHENTICATE SOCKET: " + token);
//fs.writeFileSync('/tmp/test-sync', token, {flag:'a+'});
(async () => { (async () => {
var isvalid = await checkToken(token); var isvalid = await checkToken(token);
if (isvalid.action == "ok") { if (isvalid.action == "ok") {
console.log("Authserver ok ", s.id + " - " + token); console.log("Authserver ok ", s.id + " - " + token);
//fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'});
// pubClient.set(session, resob1string, function(err, res) { // pubClient.set(session, resob1string, function(err, res) {
// }); // });
usersession.SOCKET.user = isvalid.user; usersession.SOCKET.user = isvalid.user;
@ -668,6 +387,7 @@ io.on("connection", (s) => {
s.auth = true; s.auth = true;
} else { } else {
console.log("Authserver no ", s.id + " - " + token); console.log("Authserver no ", s.id + " - " + token);
//fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'});
s.auth = false; s.auth = false;
} }
})(); })();
@ -679,6 +399,8 @@ io.on("connection", (s) => {
user: data, user: data,
id: s.id, id: s.id,
}; };
var xxx = JSON.stringify(binddata);
//fs.writeFileSync('/tmp/test-syncx', xxx, {flag:'a+'});
checkstream(binddata); checkstream(binddata);
}); });

20
readmongo/swarmlab-app/src/run/db.js
View File

@ -0,0 +1,20 @@
const MongoClient = require('mongodb').MongoClient;
const dotenv = require("dotenv");
const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env;
let db;
const loadDB = async () => {
if (db) {
return db;
}
try {
const client = await MongoClient.connect(URL, { useNewUrlParser: true, useUnifiedTopology: true });
db = client.db(DATABASE);
} catch (err) {
console.error(err);
}
return db;
};
module.exports = loadDB;
Write Preview
Loading…
Cancel
Save