Browse Source

edit readmongo

master
zeus 3 years ago
parent
commit
279d58a51e
  1. 418
      readmongo/swarmlab-app/src/run/app.js
  2. 20
      readmongo/swarmlab-app/src/run/db.js

418
readmongo/swarmlab-app/src/run/app.js

@ -9,6 +9,8 @@ const dotenv = require("dotenv");
const check123 = dotenv.config();
const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env;
const fs = require('fs');
const io = require("socket.io")(http, {
// pingTimeout: 30000,
// allowUpgrades: false,
@ -51,6 +53,10 @@ pubClient.on("connect", function () {
const MongoClient = require("mongodb").MongoClient;
const { DateTime } = require("luxon");
const loadDB = require('./db');
var async = require("async");
const { check, validationResult } = require("express-validator");
const urlExistSync = require("url-exist-sync");
@ -103,104 +109,26 @@ const corsOptions = {
// ------------------------------
// Lefos - get length of logs
app.get("/length", cors(corsOptions), (req, res) => {
console.error("getting length of logs");
var url = URL;
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
let containers = [];
var type;
var found = 0;
obj.forEach((value) => {
//new code
if (containers.length == 0) {
type = checkService();
if (type == "out") {
var RES = new Object();
var containers = [];
(async() => {
var RES = new Object()
var RESL = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
RES.data.forEach((value) => {
containers.push({
name: value.container_name,
name: value,
lengtho: 1,
lengthe: 0,
});
} else if (type == "err") {
containers.push({
name: value.container_name,
lengtho: 0,
lengthe: 1,
});
}
} else {
containers.forEach((val) => {
if (val.name == value.container_name) {
found = 1;
type = checkService();
if (type == "out") {
val.lengtho++;
} else if (type == "err") {
val.lengthe++;
}
}
});
if (found == 0) {
type = checkService();
if (type == "out") {
containers.push({
name: value.container_name,
lengtho: 1,
lengthe: 0,
});
} else if ("error") {
containers.push({
name: value.container_name,
lengtho: 0,
lengthe: 1,
});
}
}
found = 0;
}
// new code end
function checkService() {
if (value.tag.includes("mongodb")) {
var tmp = JSON.parse(value.log);
if (tmp.s == "I") return "out";
// count every informative log as an output log
else if (tmp.s == "W" || tmp.s == "E") return "err"; //count every warning and error log
} else if (value.tag.includes("node")) {
if (IsJsonString(value.log)) var test = JSON.parse(value.log);
else {
if (value.source == "stdout") return "out";
else return "err";
}
var type = test.type;
if (type == "out") return "out";
else return "err";
} else if (value.tag.includes("redis")) {
return "out";
}
}
});
var RES = new Object();
RES.data = containers;
// console.log("Sending Data: " + jsonfinal);
res.json(containers);
db.close();
});
}
);
});
RESL.data = containers;
res.json(RESL)
})()
});
// έλεγχος αν η δομή είναι κατάλληλη για μετατροπή σε json
@ -216,74 +144,26 @@ function IsJsonString(str) {
// endpoint για την αναγνώριση των υπηρεσιών στο δίκτυο
app.get("/services", cors(corsOptions), (req, res) => {
console.error("getting length of logs");
var url = URL;
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
var services = [];
var flag = false;
obj.forEach((value) => {
if (flag == false) {
services.push(value.container_name);
flag = true;
} else {
var tmp = "none";
var count = 0;
services.forEach((val) => {
var length = services.length;
var found = false;
if (
val != value.container_name &&
count + 1 < length &&
found == false
) {
count++;
//services.push(value.container_name)
} else if (
val != value.container_name &&
count + 1 == length &&
found == false
) {
//found =true;
tmp = value.container_name;
} else if (val == value.container_name) {
found = true;
}
});
if (tmp != "none") {
services.push(tmp);
}
}
});
var RES = new Object();
RES.data = services;
// console.log("Sending Data: " + jsonfinal);
res.json(RES.data);
db.close();
});
}
);
(async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
RES.data.forEach((value) => {
jsonfinal.push(value.tailed_path);
});
//var services = Object.keys(jsonfinal);
var uniq = [...new Set(jsonfinal)];
res.json(uniq)
})()
});
// Lefos - mongo test read
app.get("/test", cors(corsOptions), (req, res) => {
console.log("reading from db....");
fs.writeFileSync('/tmp/test-sync', 'db', {flag:'a+'});
var url = URL;
MongoClient.connect(
@ -303,20 +183,13 @@ app.get("/test", cors(corsOptions), (req, res) => {
var mongo = [];
var app = [];
obj.forEach((value) => {
if (value.container_name == "/redisserver") {
// jsonfinal.push(JSON.parse(value.log));
jsonfinal.push(value);
} else if (value.container_name == "/mongo") {
mongo.push(JSON.parse(value.log));
} else if (value.log.includes("app_name")) {
app.push(JSON.parse(value.log));
}
});
var RES = new Object();
RES.data = {
redis: jsonfinal,
mongo: mongo,
app: app,
redis: '',
mongo: '',
app: jsonfinal
};
res.json(RES.data);
db.close();
@ -329,34 +202,20 @@ app.get("/test", cors(corsOptions), (req, res) => {
app.get("/raw", cors(corsOptions), (req, res) => {
console.log("reading from db....");
var url = URL;
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
var jsonfinal = [];
obj.forEach((value) => {
jsonfinal.push(value);
});
var RES = new Object();
RES.data = {
// fs.writeFileSync('/tmp/test-sync', 'db1', {flag:'a+'});
(async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
RES.data.forEach((value) => {
jsonfinal.push(value.tailed_path);
});
RES.src = {
all: jsonfinal,
};
res.json(RES.data);
db.close();
});
}
);
res.json(RES.src)
})()
});
// βασικό endpoint που επιστρέφει κατάλληλα τα logs στην ευρετηρίαση του web-client
@ -379,162 +238,22 @@ app.get("/test2", cors(corsOptions), (req, res) => {
console.error("EXTRA PARAMS: " + logtext);
//console.log("TEST LOG");
var url = URL;
(async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
obj.forEach((value) => {
var test = IsJsonString(value.log);
if (value.tag.includes("node") && IsJsonString(value.log)) {
var tmp = JSON.parse(value.log);
var container_name = value.container_name;
var tmptype;
tmp.app_name = container_name;
if (tmp.type == "out") {
tmptype = '<div class="outtype" >' + tmp.type + "</div>";
} else if (tmp.type == "err") {
tmptype = '<div class="errtype" >' + tmp.type + "</div>";
}
var tmp2 = {
message: tmp.message,
timestamp: value.time,
type: tmptype,
process_id: "-",
app_name: value.container_name,
};
// filtrarisma gia to text poy exei dwsei o xrhsths
if (
logtext != "" &&
tmp.message.includes(logtext) &&
(container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
} else if (logtext != "" && !tmp.message.includes(logtext)) {
} else if (
logtext == "" &&
(container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
}
} else if (value.tag.includes("node") && test == false) {
//console.error("EDW EIMAI!!!!");
var tmptype;
if (value.source == "stdout") {
tmptype = '<div class="outtype" >' + "out" + "</div>";
} else if (value.source == "stderr") {
tmptype = '<div class="errtype" >' + "err" + "</div>";
}
var tmp2 = {
message: value.log,
timestamp: value.time,
type: tmptype,
process_id: "-",
app_name: value.container_name,
};
if (
logtext != "" &&
tmp2.message.includes(logtext) &&
(value.container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
} else if (logtext != "" && !value.log.includes(logtext)) {
//console.error("lathos sinthiki");
} else if (
logtext == "" &&
(value.container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
}
} else if (
value.tag.includes("redis") &&
(value.container_name == selected || selected == "All")
) {
var tmplog = value;
//timestamp
var time = tmplog.time;
var tmp = {
message: tmplog.log,
timestamp: time,
type: "<div class= 'outtype'>out</div>",
process_id: "-",
app_name: value.container_name,
};
if (logtext == "" || tmp.message.includes(logtext))
jsonfinal.push(tmp);
} else if (
value.tag.includes("mongodb") &&
(value.container_name == selected || selected == "All")
) {
var tmplog = JSON.parse(value.log);
var msg2 = tmplog.msg;
//timestam
var time = value.time;
RES.data.forEach((value) => {
var tmp = {
message: msg2,
timestamp: time,
type: "<div class= 'outtype'>out</div>",
process_id: "-",
app_name: value.container_name,
};
if (logtext == "" || tmp.message.includes(logtext))
jsonfinal.push(tmp);
} else {
var tmp = {
message: value.log,
message: value.message,
timestamp: value.time,
type: "<div class= 'outtype'>" + value.source + "</div>",
process_id: "-",
app_name: value.container_name,
type: "<div class= 'outtype'>" + value._id+ "</div>",
process_id: '',
app_name: value.tailed_path,
};
}
});
//console.log("Sending Data: " + jsonfinal);
if (sortname == "type") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.type.localeCompare(b.type);
} else {
return b.type.localeCompare(a.type);
}
});
} else if (sortname == "message") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.message.localeCompare(b.message);
} else {
return b.message.localeCompare(a.message);
}
});
} else if (sortname == "time") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.timestamp.localeCompare(b.timestamp);
} else {
return b.timestamp.localeCompare(a.timestamp);
}
});
} else if (sortname == "app_name") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.app_name.localeCompare(b.app_name);
} else {
return b.app_name.localeCompare(a.app_name);
}
});
}
jsonfinal.push(tmp);
});
//Pagination
var total = jsonfinal.length;
var perpage = per_page;
@ -584,13 +303,10 @@ app.get("/test2", cors(corsOptions), (req, res) => {
// console.log("Sending Data: " + jsonfinal);
res.json(RES.data);
db.close();
});
}
);
})()
});
console.log("reading from db....");
// συνάρτηση που χρησιμοποιείται ως callback για την δημιουργία stream με τη βάση και συγκεκριμένο user
async function onCollectionNew(err, collection) {
@ -639,6 +355,7 @@ async function onCollectionNew(err, collection) {
// sockets και διαχείριση συνδέσεων
io.on("connection", (s) => {
console.error("socket connection");
//fs.writeFileSync('/tmp/test-sync', 'Hey there!', {flag:'a+'});
var usersession = new Object();
usersession.SOCKET = {};
@ -652,10 +369,12 @@ io.on("connection", (s) => {
s.on("authenticate", function (data) {
const token = data;
console.log("TEST LOG INSIDE ATHENTICATE SOCKET: " + token);
//fs.writeFileSync('/tmp/test-sync', token, {flag:'a+'});
(async () => {
var isvalid = await checkToken(token);
if (isvalid.action == "ok") {
console.log("Authserver ok ", s.id + " - " + token);
//fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'});
// pubClient.set(session, resob1string, function(err, res) {
// });
usersession.SOCKET.user = isvalid.user;
@ -668,6 +387,7 @@ io.on("connection", (s) => {
s.auth = true;
} else {
console.log("Authserver no ", s.id + " - " + token);
//fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'});
s.auth = false;
}
})();
@ -679,6 +399,8 @@ io.on("connection", (s) => {
user: data,
id: s.id,
};
var xxx = JSON.stringify(binddata);
//fs.writeFileSync('/tmp/test-syncx', xxx, {flag:'a+'});
checkstream(binddata);
});

20
readmongo/swarmlab-app/src/run/db.js

@ -0,0 +1,20 @@
const MongoClient = require('mongodb').MongoClient;
const dotenv = require("dotenv");
const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env;
let db;
const loadDB = async () => {
if (db) {
return db;
}
try {
const client = await MongoClient.connect(URL, { useNewUrlParser: true, useUnifiedTopology: true });
db = client.db(DATABASE);
} catch (err) {
console.error(err);
}
return db;
};
module.exports = loadDB;
Loading…
Cancel
Save