swarmlab
/
poc-datacollector
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

edit readmongo

master
zeus 3 years ago
parent
08a015a922
commit
279d58a51e
2 changed files with 90 additions and 348 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 416
      readmongo/swarmlab-app/src/run/app.js
  2. 20
      readmongo/swarmlab-app/src/run/db.js

416
readmongo/swarmlab-app/src/run/app.js
View File

@ -9,6 +9,8 @@ const dotenv = require("dotenv");
const check123 = dotenv.config();
const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env;
const fs = require('fs');
const io = require("socket.io")(http, {
// pingTimeout: 30000,
// allowUpgrades: false,
@ -51,6 +53,10 @@ pubClient.on("connect", function () {
const MongoClient = require("mongodb").MongoClient;
const { DateTime } = require("luxon");
const loadDB = require('./db');
var async = require("async");
const { check, validationResult } = require("express-validator");
const urlExistSync = require("url-exist-sync");
@ -103,104 +109,26 @@ const corsOptions = {
// ------------------------------
// Lefos - get length of logs
app.get("/length", cors(corsOptions), (req, res) => {
console.error("getting length of logs");
var url = URL;
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
let containers = [];
var type;
var found = 0;
obj.forEach((value) => {
//new code
if (containers.length == 0) {
type = checkService();
if (type == "out") {
var RES = new Object();
var containers = [];
(async() => {
var RES = new Object()
var RESL = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
RES.data.forEach((value) => {
containers.push({
name: value.container_name,
name: value,
lengtho: 1,
lengthe: 0,
});
} else if (type == "err") {
containers.push({
name: value.container_name,
lengtho: 0,
lengthe: 1,
});
}
} else {
containers.forEach((val) => {
if (val.name == value.container_name) {
found = 1;
type = checkService();
if (type == "out") {
val.lengtho++;
} else if (type == "err") {
val.lengthe++;
}
}
});
if (found == 0) {
type = checkService();
if (type == "out") {
containers.push({
name: value.container_name,
lengtho: 1,
lengthe: 0,
});
} else if ("error") {
containers.push({
name: value.container_name,
lengtho: 0,
lengthe: 1,
});
}
}
found = 0;
}
// new code end
function checkService() {
if (value.tag.includes("mongodb")) {
var tmp = JSON.parse(value.log);
if (tmp.s == "I") return "out";
// count every informative log as an output log
else if (tmp.s == "W" || tmp.s == "E") return "err"; //count every warning and error log
} else if (value.tag.includes("node")) {
if (IsJsonString(value.log)) var test = JSON.parse(value.log);
else {
if (value.source == "stdout") return "out";
else return "err";
}
var type = test.type;
if (type == "out") return "out";
else return "err";
} else if (value.tag.includes("redis")) {
return "out";
}
}
});
var RES = new Object();
RES.data = containers;
// console.log("Sending Data: " + jsonfinal);
res.json(containers);
db.close();
});
}
);
});
RESL.data = containers;
res.json(RESL)
})()
});
// έλεγχος αν η δομή είναι κατάλληλη για μετατροπή σε json
@ -217,73 +145,25 @@ function IsJsonString(str) {
app.get("/services", cors(corsOptions), (req, res) => {
console.error("getting length of logs");
var url = URL;
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
var services = [];
var flag = false;
obj.forEach((value) => {
if (flag == false) {
services.push(value.container_name);
flag = true;
} else {
var tmp = "none";
var count = 0;
services.forEach((val) => {
var length = services.length;
var found = false;
if (
val != value.container_name &&
count + 1 < length &&
found == false
) {
count++;
//services.push(value.container_name)
} else if (
val != value.container_name &&
count + 1 == length &&
found == false
) {
//found =true;
tmp = value.container_name;
} else if (val == value.container_name) {
found = true;
}
});
if (tmp != "none") {
services.push(tmp);
}
}
});
var RES = new Object();
RES.data = services;
// console.log("Sending Data: " + jsonfinal);
res.json(RES.data);
db.close();
});
}
);
(async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
RES.data.forEach((value) => {
jsonfinal.push(value.tailed_path);
});
//var services = Object.keys(jsonfinal);
var uniq = [...new Set(jsonfinal)];
res.json(uniq)
})()
});
// Lefos - mongo test read
app.get("/test", cors(corsOptions), (req, res) => {
console.log("reading from db....");
fs.writeFileSync('/tmp/test-sync', 'db', {flag:'a+'});
var url = URL;
MongoClient.connect(
@ -303,20 +183,13 @@ app.get("/test", cors(corsOptions), (req, res) => {
var mongo = [];
var app = [];
obj.forEach((value) => {
if (value.container_name == "/redisserver") {
// jsonfinal.push(JSON.parse(value.log));
jsonfinal.push(value);
} else if (value.container_name == "/mongo") {
mongo.push(JSON.parse(value.log));
} else if (value.log.includes("app_name")) {
app.push(JSON.parse(value.log));
}
});
var RES = new Object();
RES.data = {
redis: jsonfinal,
mongo: mongo,
app: app,
redis: '',
mongo: '',
app: jsonfinal
};
res.json(RES.data);
db.close();
@ -329,34 +202,20 @@ app.get("/test", cors(corsOptions), (req, res) => {
app.get("/raw", cors(corsOptions), (req, res) => {
console.log("reading from db....");
var url = URL;
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
var jsonfinal = [];
obj.forEach((value) => {
jsonfinal.push(value);
});
var RES = new Object();
RES.data = {
// fs.writeFileSync('/tmp/test-sync', 'db1', {flag:'a+'});
(async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
RES.data.forEach((value) => {
jsonfinal.push(value.tailed_path);
});
RES.src = {
all: jsonfinal,
};
res.json(RES.data);
db.close();
});
}
);
res.json(RES.src)
})()
});
// βασικό endpoint που επιστρέφει κατάλληλα τα logs στην ευρετηρίαση του web-client
@ -379,162 +238,22 @@ app.get("/test2", cors(corsOptions), (req, res) => {
console.error("EXTRA PARAMS: " + logtext);
//console.log("TEST LOG");
var url = URL;
(async() => {
var RES = new Object()
const mydb = await loadDB()
RES.data = await mydb.collection(COLLECTION).find({}).toArray()
var jsonfinal = [];
MongoClient.connect(
url,
{ useNewUrlParser: true, useUnifiedTopology: true },
function (err, db) {
if (err) throw err;
var dbo = db.db(DATABASE);
dbo
.collection(COLLECTION)
.find({})
.toArray(function (err, result) {
if (err) throw err;
// EPIDI EXW NESTED JSON PREPEI NA TO KANW PARSE DUO FORES
var obj = JSON.parse(JSON.stringify(result));
obj.forEach((value) => {
var test = IsJsonString(value.log);
if (value.tag.includes("node") && IsJsonString(value.log)) {
var tmp = JSON.parse(value.log);
var container_name = value.container_name;
var tmptype;
tmp.app_name = container_name;
if (tmp.type == "out") {
tmptype = '<div class="outtype" >' + tmp.type + "</div>";
} else if (tmp.type == "err") {
tmptype = '<div class="errtype" >' + tmp.type + "</div>";
}
var tmp2 = {
message: tmp.message,
timestamp: value.time,
type: tmptype,
process_id: "-",
app_name: value.container_name,
};
// filtrarisma gia to text poy exei dwsei o xrhsths
if (
logtext != "" &&
tmp.message.includes(logtext) &&
(container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
} else if (logtext != "" && !tmp.message.includes(logtext)) {
} else if (
logtext == "" &&
(container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
}
} else if (value.tag.includes("node") && test == false) {
//console.error("EDW EIMAI!!!!");
var tmptype;
if (value.source == "stdout") {
tmptype = '<div class="outtype" >' + "out" + "</div>";
} else if (value.source == "stderr") {
tmptype = '<div class="errtype" >' + "err" + "</div>";
}
var tmp2 = {
message: value.log,
timestamp: value.time,
type: tmptype,
process_id: "-",
app_name: value.container_name,
};
if (
logtext != "" &&
tmp2.message.includes(logtext) &&
(value.container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
} else if (logtext != "" && !value.log.includes(logtext)) {
//console.error("lathos sinthiki");
} else if (
logtext == "" &&
(value.container_name == selected || selected == "All")
) {
jsonfinal.push(tmp2);
}
} else if (
value.tag.includes("redis") &&
(value.container_name == selected || selected == "All")
) {
var tmplog = value;
//timestamp
var time = tmplog.time;
var tmp = {
message: tmplog.log,
timestamp: time,
type: "<div class= 'outtype'>out</div>",
process_id: "-",
app_name: value.container_name,
};
if (logtext == "" || tmp.message.includes(logtext))
jsonfinal.push(tmp);
} else if (
value.tag.includes("mongodb") &&
(value.container_name == selected || selected == "All")
) {
var tmplog = JSON.parse(value.log);
var msg2 = tmplog.msg;
//timestam
var time = value.time;
RES.data.forEach((value) => {
var tmp = {
message: msg2,
timestamp: time,
type: "<div class= 'outtype'>out</div>",
process_id: "-",
app_name: value.container_name,
};
if (logtext == "" || tmp.message.includes(logtext))
jsonfinal.push(tmp);
} else {
var tmp = {
message: value.log,
message: value.message,
timestamp: value.time,
type: "<div class= 'outtype'>" + value.source + "</div>",
process_id: "-",
app_name: value.container_name,
type: "<div class= 'outtype'>" + value._id+ "</div>",
process_id: '',
app_name: value.tailed_path,
};
}
});
//console.log("Sending Data: " + jsonfinal);
if (sortname == "type") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.type.localeCompare(b.type);
} else {
return b.type.localeCompare(a.type);
}
});
} else if (sortname == "message") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.message.localeCompare(b.message);
} else {
return b.message.localeCompare(a.message);
}
});
} else if (sortname == "time") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.timestamp.localeCompare(b.timestamp);
} else {
return b.timestamp.localeCompare(a.timestamp);
}
});
} else if (sortname == "app_name") {
jsonfinal.sort(function (a, b) {
if (sortorder == "asc") {
return a.app_name.localeCompare(b.app_name);
} else {
return b.app_name.localeCompare(a.app_name);
}
});
}
jsonfinal.push(tmp);
});
//Pagination
var total = jsonfinal.length;
var perpage = per_page;
@ -584,13 +303,10 @@ app.get("/test2", cors(corsOptions), (req, res) => {
// console.log("Sending Data: " + jsonfinal);
res.json(RES.data);
db.close();
});
}
);
})()
});
console.log("reading from db....");
// συνάρτηση που χρησιμοποιείται ως callback για την δημιουργία stream με τη βάση και συγκεκριμένο user
async function onCollectionNew(err, collection) {
@ -639,6 +355,7 @@ async function onCollectionNew(err, collection) {
// sockets και διαχείριση συνδέσεων
io.on("connection", (s) => {
console.error("socket connection");
//fs.writeFileSync('/tmp/test-sync', 'Hey there!', {flag:'a+'});
var usersession = new Object();
usersession.SOCKET = {};
@ -652,10 +369,12 @@ io.on("connection", (s) => {
s.on("authenticate", function (data) {
const token = data;
console.log("TEST LOG INSIDE ATHENTICATE SOCKET: " + token);
//fs.writeFileSync('/tmp/test-sync', token, {flag:'a+'});
(async () => {
var isvalid = await checkToken(token);
if (isvalid.action == "ok") {
console.log("Authserver ok ", s.id + " - " + token);
//fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'});
// pubClient.set(session, resob1string, function(err, res) {
// });
usersession.SOCKET.user = isvalid.user;
@ -668,6 +387,7 @@ io.on("connection", (s) => {
s.auth = true;
} else {
console.log("Authserver no ", s.id + " - " + token);
//fs.writeFileSync('/tmp/test-sync', s.id, {flag:'a+'});
s.auth = false;
}
})();
@ -679,6 +399,8 @@ io.on("connection", (s) => {
user: data,
id: s.id,
};
var xxx = JSON.stringify(binddata);
//fs.writeFileSync('/tmp/test-syncx', xxx, {flag:'a+'});
checkstream(binddata);
});

20
readmongo/swarmlab-app/src/run/db.js
View File

@ -0,0 +1,20 @@
const MongoClient = require('mongodb').MongoClient;
const dotenv = require("dotenv");
const { DATABASE, COLLECTION, URL, REDIS, REDIS_PORT, PORT } = process.env;
let db;
const loadDB = async () => {
if (db) {
return db;
}
try {
const client = await MongoClient.connect(URL, { useNewUrlParser: true, useUnifiedTopology: true });
db = client.db(DATABASE);
} catch (err) {
console.error(err);
}
return db;
};
module.exports = loadDB;
Write Preview
Loading…
Cancel
Save