From 9d5a94d98cac8f089ef79a8711027da8299035c1 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 7 Jan 2019 10:33:43 -0600 Subject: [PATCH] add diskusage monitor --- .gitignore | 3 +- RuleEngine.js | 5 ++ .../diskUsageMonitor.config.json.example | 9 +++ diskUsageMonitor/diskUsageMonitor.js | 62 +++++++++++++++++++ logAggregator.js | 2 +- package-lock.json | 19 ++++++ package.json | 1 + rules/diskusage.rule.json | 5 ++ 8 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 diskUsageMonitor/diskUsageMonitor.config.json.example create mode 100644 diskUsageMonitor/diskUsageMonitor.js create mode 100644 rules/diskusage.rule.json diff --git a/.gitignore b/.gitignore index 10a639a..9341011 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,5 @@ typings/ # DynamoDB Local files .dynamodb/ casRpsLogExtractor/casRpsLogExtractor.config.json -casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat \ No newline at end of file +casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat +diskUsageMonitor/diskUsageMonitor.config.json diff --git a/RuleEngine.js b/RuleEngine.js index dc51b30..5bbc91a 100644 --- a/RuleEngine.js +++ b/RuleEngine.js @@ -45,6 +45,7 @@ exports.RuleEngine = class RuleEngine { processData(inputLine) { let inputData = inputLine.data; logger.log("debug", "ProcessingType " + inputLine.processingType); + logger.log("debug", "InputLine " + JSON.stringify(inputLine)); let allWhiteFilterRules = this.ruleSet[inputLine.processingType].rules; if (this.processLogicOr(allWhiteFilterRules, inputData)) { return (true); } logger.log("debug", "No whitelisting matched; returning false"); @@ -103,11 +104,15 @@ exports.RuleEngine = class RuleEngine { } processLogicFlat(theWhiteFilterRule, inputData) { logger.log("debug", "Logic FLAT start - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule)); + logger.log("debug", "inputData: " + JSON.stringify(inputData)); let isKnownComp = false; let retVal = false; if (theWhiteFilterRule.Comp === "===") { isKnownComp = true; if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) {retVal = true;} + } else if (theWhiteFilterRule.Comp === "!==") { + isKnownComp = true; + if (inputData[theWhiteFilterRule.Field] !== theWhiteFilterRule.Value) {retVal = true;} } else if (theWhiteFilterRule.Comp === "<") { isKnownComp = true; if (inputData[theWhiteFilterRule.Field] < theWhiteFilterRule.Value) {retVal = true;} diff --git a/diskUsageMonitor/diskUsageMonitor.config.json.example b/diskUsageMonitor/diskUsageMonitor.config.json.example new file mode 100644 index 0000000..e5525a0 --- /dev/null +++ b/diskUsageMonitor/diskUsageMonitor.config.json.example @@ -0,0 +1,9 @@ +{ + "sourceTag": "CCE1DiskUsage", + "serverURI": "http://localhost:2222", + "pollInterval": 3600000, + "diskLimits":[ + {"path":"C:\\", "minFree":"1"}, + {"path":"T:\\", "minFree":"1"} + ] +} \ No newline at end of file diff --git a/diskUsageMonitor/diskUsageMonitor.js b/diskUsageMonitor/diskUsageMonitor.js new file mode 100644 index 0000000..9ce00b9 --- /dev/null +++ b/diskUsageMonitor/diskUsageMonitor.js @@ -0,0 +1,62 @@ +const request = require("request-promise"); +const fs = require("fs"); +const winston = require("winston"); +const disk = require('diskusage'); + +const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf(info => { + return `${info.timestamp} ${info.level}: ${info.message}`; + }) + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ filename: './log/diskUsageMonitor.log', level: 'error' }), + new winston.transports.File({ filename: './log/Combined.log' }) + ] +}); + +const processingType = "diskusage"; + +logger.log("info", "Starting diskUsageMonitor v0.1 by j.tretter@gmail.com"); +const config = JSON.parse(fs.readFileSync("diskUsageMonitor.config.json", "utf8")); +logger.log("debug", "Config" + JSON.stringify(config)); + +function cb(){ + config.diskLimits.forEach(theDiskLimit=>{ + theMessage={}; + theMessage.processingType=processingType; + theMessage.sourceTag=config.sourceTag; + theMessage.timestamp=new Date(); + theDiskLimit.isWarn=false; + disk.check(theDiskLimit.path, function(err, info) { + if (err) { + logger.log("error","Error reading disk Information " + err); + theDiskLimit.error = err; + theDiskLimit.isWarn = true; + } else { + let availGB=info.available/1024/1024; + let totalGB=info.total/1024/1024; + theDiskLimit.availGB=availGB; + theDiskLimit.totalGB=totalGB; + if (availGB<=theDiskLimit.minFree) { + theDiskLimit.isWarn=true; + } + } + theMessage.data=theDiskLimit; + logger.log("debug","Sending Message: " + JSON.stringify(theMessage)); + + request({ method: 'POST', uri: config.serverURI, body: JSON.stringify([theMessage]), json: true }).then(_ => { + logger.log("debug","Sent Request to Service"); + }).catch(err=>{ + logger.log("error","Sending Request to Service: " + err); + }); + + }); + }) +} + +cb(); +setInterval(cb,config.pollInterval); diff --git a/logAggregator.js b/logAggregator.js index 8526425..955edd1 100644 --- a/logAggregator.js +++ b/logAggregator.js @@ -37,7 +37,7 @@ const server = micro(async (req, res) => { let theInputLine=allInputLines[i]; let isWhiteListed = ruleEngine.processData(theInputLine); let miniInput = jsonminify(JSON.stringify(theInputLine)); - let now = Date(); + if (isWhiteListed) { logger.log("debug", "Not Whitelisted: " + miniInput); fs.appendFileSync("./log/skipped.log", miniInput + "\n"); diff --git a/package-lock.json b/package-lock.json index ef5ac24..0c1b651 100644 --- a/package-lock.json +++ b/package-lock.json @@ -229,6 +229,15 @@ "kuler": "1.0.x" } }, + "diskusage": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/diskusage/-/diskusage-1.0.0.tgz", + "integrity": "sha512-eOrPVSZLxo8kXz03GA+gdKauSSmicNET/pXMZKXs1vkdZrEayeZmJYu1Eo2uzVhkQL748vq1irIi7hzoPWBRmQ==", + "requires": { + "es6-promise": "^4.2.5", + "nan": "^2.11.1" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -251,6 +260,11 @@ "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", "integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA==" }, + "es6-promise": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", + "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==" + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -505,6 +519,11 @@ "tedious": "^2.7.1" } }, + "nan": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz", + "integrity": "sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==" + }, "native-duplexpair": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/native-duplexpair/-/native-duplexpair-1.0.0.tgz", diff --git a/package.json b/package.json index 43a4a8c..12559d8 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "author": "", "license": "ISC", "dependencies": { + "diskusage": "^1.0.0", "jsonminify": "^0.4.1", "micro": "^9.3.3", "msnodesqlv8": "^0.6.11", diff --git a/rules/diskusage.rule.json b/rules/diskusage.rule.json new file mode 100644 index 0000000..57e9aef --- /dev/null +++ b/rules/diskusage.rule.json @@ -0,0 +1,5 @@ +{ "diskusage": + { "rules":[ + { "Logic":"-", "Field":"isWarn", "Comp":"!==", "Value":true} + ]} + } \ No newline at end of file