diff --git a/casRpsLogExtractor/casRpsLogExtractor.js b/casRpsLogExtractor/casRpsLogExtractor.js index e652744..fc4868f 100644 --- a/casRpsLogExtractor/casRpsLogExtractor.js +++ b/casRpsLogExtractor/casRpsLogExtractor.js @@ -1,5 +1,5 @@ -const sql = require("mssql/msnodesqlv8"); -//const sql = require("mssql"); +//const sql = require("mssql/msnodesqlv8"); +const sql = require("mssql"); const request = require("request-promise"); const fs = require("fs"); const winston = require("winston"); @@ -25,7 +25,9 @@ const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf logger.log("debug", "Config" + JSON.stringify(config)) const processingType = "casrpslog"; let maxPKey = ""; +let maxPKeyUnsent = ""; let lastOutMaxPKey = ""; +let allRows = []; try { lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8"); logger.log("debug", "MaxPKey Loaded: " + maxPKey); @@ -53,20 +55,28 @@ sql.connect(config.sqlConfig, err => { process.stdout.write("#"); request.json = true; - logger.log('debug', "sending data:" + JSON.stringify(outData)); - request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(outData), json: true }).then(_ => { - // count up the maxPKey only when the request was successful... - if (row.PKey > maxPKey) { - maxPKey = row.PKey; - fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey); - logger.log("debug", "MaxPKey Written: " + maxPKey); - } - }).catch(err=>{ - logger.log("error","Sending Request to Service: " + err) - }); + logger.log('debug', "adding row to array:" + JSON.stringify(outData)); + allRows.push(outData); + + if (row.PKey > maxPKeyUnsent) { + maxPKeyUnsent = row.PKey; + logger.log("debug", "MaxPKeyUnsent Written: " + maxPKeyUnsent); + } }); sqlRequest.on('done', result => { + if (allRows.length>0){ + request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => { + // count up the maxPKey only when the request was successful... + maxPKey=maxPKeyUnsent; + fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey); + }).catch(err=>{ + logger.log("error","Sending Request to Service: " + err) + }); + process.stdout.write(">" + allRows.length + ">"); + allRows=[]; + } + if (lastOutMaxPKey !== maxPKey) { process.stdout.write("[" + maxPKey + "]\n"); lastOutMaxPKey = maxPKey; diff --git a/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat new file mode 100644 index 0000000..df706b3 --- /dev/null +++ b/casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat @@ -0,0 +1 @@ +0010000000a87kj9 \ No newline at end of file diff --git a/logAggregator.js b/logAggregator.js index 71a8f4e..a0a7e6f 100644 --- a/logAggregator.js +++ b/logAggregator.js @@ -27,23 +27,27 @@ const server = micro(async (req, res) => { const js = await json(req); try { - inputLine = JSON.parse(js); + allInputLines = JSON.parse(js); + send(res, 200, "OK"); } catch (ex) { - logger.log("error", "Input not in JSON format:" + line); + logger.log("error", "Input not in JSON format:" + js); + send(res, 401, "Input not in JSON Format"); } - let isWhiteListed = ruleEngine.processData(inputLine); - let miniInput = jsonminify(JSON.stringify(inputLine)); - let now = Date(); - if (isWhiteListed) { - logger.log("debug", "Not Whitelisted: " + miniInput); - fs.appendFileSync("./log/skipped.log", miniInput + "\n"); - } else { - logger.log("info", "Not Whitelisted: " + miniInput); - fs.appendFileSync("./log/issues.log", miniInput + "\n"); + for(i=0;i