joe/logAggregator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more improvements

Browse Source
This commit is contained in:
Tretter
2019-01-04 12:46:40 -06:00
parent e45e525895
commit f1cc449113
6 changed files with 81 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
RuleEngine.js
View File

@@ -16,14 +16,35 @@ const logger = winston.createLogger({
]
});
exports.RuleEngine = class RuleEngine {
constructor(ruleSetFileMask) {
this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFileMask + '.rule.json', 'utf8'));
constructor(ruleSetFiles) {
logger.log("info", "Instanciated RulesEngine - RuleSetFiles: " + ruleSetFiles);
this.ruleSet = {};
if (ruleSetFiles === "*") {
fs.readdirSync("./rules/").forEach(file => {
if (this.reTest("/.*\.rule\.json$/", file)) {
logger.log("info", "Loading ruleset file " + file);
let newRuleSet = JSON.parse(fs.readFileSync('./rules/' + file, 'utf8'));
logger.log("info", "Adding ruleset " + Object.keys(newRuleSet)[0]);
this.ruleSet[Object.keys(newRuleSet)[0]] = newRuleSet[Object.keys(newRuleSet)[0]]
}
});
} else if (ruleSetFiles.indexOf(",") > 1) {
ruleSetFiles.split(",").forEach(file => {
logger.log("info", "Loading ruleset file " + file);
let newRuleSet = JSON.parse(fs.readFileSync('./rules/' + file + '.rule.json', 'utf8'));
logger.log("info", "Adding ruleset " + Object.keys(newRuleSet)[0]);
this.ruleSet[Object.keys(newRuleSet)[0]] = newRuleSet[Object.keys(newRuleSet)[0]]
});
} else {
logger.log("info", "Loading ruleset file " + ruleSetFiles);
this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFiles + '.rule.json', 'utf8'));
logger.log("info", "Adding Ruleset " + Object.keys(this.ruleSet)[0]);
}
}
processData(inputLine) {
let inputData = inputLine.data;
logger.log("debug", "ProcessingType " + inputLine.processingType + " RuleSet: " + this.ruleSet);
logger.log("debug", "ProcessingType " + inputLine.processingType);
let allWhiteFilterRules = this.ruleSet[inputLine.processingType].rules;
if (this.processLogicOr(allWhiteFilterRules, inputData)) { return (true); }
logger.log("debug", "No whitelisting matched; returning false");
@@ -59,12 +80,12 @@ exports.RuleEngine = class RuleEngine {
logger.log("debug", "Logic OR - start");
for (let i = 0; i < allWhiteFilterRules.length; i++) {
let theWhiteFilterRule = allWhiteFilterRules[i];
logger.log("debug", " Logic OR - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule));
logger.log("debug", " Logic OR - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule));
if (theWhiteFilterRule.Logic === "-") {
if (this.processLogicFlat(theWhiteFilterRule, inputData)) {
logger.log("debug", " Logic OR - FLAT - RETURNING OUT");
return (true);
};
return (true);
};
} else if (theWhiteFilterRule.Logic === "|") {
if (this.processLogicOr(theWhiteFilterRule.rules, inputData)) {
logger.log("debug", " Logic OR - OR - RETURNING OUT");
@@ -78,26 +99,42 @@ exports.RuleEngine = class RuleEngine {
}
};
logger.log("debug", "Logic OR - end");
return(false);
return (false);
}
processLogicFlat(theWhiteFilterRule, inputData) {
logger.log("debug", "Logic FLAT start - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule));
logger.log("debug", "Logic FLAT start - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule));
let isKnownComp = false;
let retVal = false;
if (theWhiteFilterRule.Comp === "===") {
if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) {
logger.log("debug", "Logic FLAT - Exiting out");
return (true);
}
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) {retVal = true;}
} else if (theWhiteFilterRule.Comp === "<") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] < theWhiteFilterRule.Value) {retVal = true;}
} else if (theWhiteFilterRule.Comp === ">") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] > theWhiteFilterRule.Value) {retVal = true;}
} else if (theWhiteFilterRule.Comp === "<=") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] <= theWhiteFilterRule.Value) {retVal = true;}
} else if (theWhiteFilterRule.Comp === ">=") {
isKnownComp = true;
if (inputData[theWhiteFilterRule.Field] >= theWhiteFilterRule.Value) {retVal = true;}
} else if (theWhiteFilterRule.Comp === "~") {
logger.log("debug","RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field]);
isKnownComp = true;
logger.log("debug", "RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field].replace(/\r/g, "\\r").replace(/\n/g, "\\n"));
retVal = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]);
}
let result = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]);
if (!isKnownComp) {
logger.log("error", "The configured Comp '" + theWhiteFilterRule.Comp + "' is unknown.")
}
if (result) {
logger.log("debug", "Logic FLAT - Exiting out");
return (true);
}
if (retVal === true) {
logger.log("debug", "Logic FLAT - Exiting out");
}
logger.log("debug", "Logic FLAT - end");
return (retVal);
}
reTest(regex, data) {

2
casRpsLogExtractor/casRpsLogExtractor.js
View File

@@ -60,6 +60,8 @@ sql.connect(config.sqlConfig, err => {
fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
logger.log("debug", "MaxPKey Written: " + maxPKey);
}
}).catch(err=>{
logger.log("error","Sending Request to Service: " + err)
});
});

2
casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat
View File

@@ -1 +1 @@
0010000000a87kiz
0010000000a87kj9

16
logAggregator.js
View File

@@ -21,26 +21,26 @@ const logger = winston.createLogger({
});
const ruleEngine = new RuleEngine("casrpslog");
logger.log("info","Service Starting.");
const ruleEngine = new RuleEngine("*");
logger.log("info", "Service Starting.");
const server = micro(async (req, res) => {
const js = await json(req);
try {
inputLine = JSON.parse(js);
} catch (ex) {
logger.log("error","Input not in JSON format:" + line);
logger.log("error", "Input not in JSON format:" + line);
}
let isWhiteListed = ruleEngine.processData(inputLine);
let miniInput=jsonminify(JSON.stringify(inputLine)) ;
let miniInput = jsonminify(JSON.stringify(inputLine));
let now = Date();
if (isWhiteListed) {
logger.log("debug","Not Whitelisted: " + miniInput);
fs.appendFileSync("./log/skipped.log",miniInput + "\n");
logger.log("debug", "Not Whitelisted: " + miniInput);
fs.appendFileSync("./log/skipped.log", miniInput + "\n");
} else {
logger.log("info","Not Whitelisted: " + miniInput);
fs.appendFileSync("./log/issues.log",miniInput + "\n");
logger.log("info", "Not Whitelisted: " + miniInput);
fs.appendFileSync("./log/issues.log", miniInput + "\n");
}
send(res, 200, "OK");

26
rules/casroslogfull.rule.json
View File

@@ -1,26 +0,0 @@
{ "casrpslog":{
"rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"},
{"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"},
{"Logic":"&", "rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"},
{"Logic":"|", "rules":[
{"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"},
{"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"}
]}
]
},
{"Logic":"&", "rules":[
{"Logic":"|", "rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"},
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"}
]},
{"Logic":"|", "rules":[
{"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."},
{"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."},
{"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"}
]}
]}
]
}
}

12
rules/casrpslog.rule.json
View File

@@ -1,16 +1,16 @@
{ "casrpslog":{
"rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"},
{"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"},
{"Logic":"&", "rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"},
{ "_comment":"Application Validation Errors",
"Logic":"&", "rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CAS 11.0"},
{"Logic":"|", "rules":[
{"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"},
{"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"}
{"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/^The promotion or tactic is not allowed to extend past the last day of the year.*/"}
]}
]
},
{"Logic":"&", "rules":[
{"_comment":"Worker Process info messages",
"Logic":"&", "rules":[
{"Logic":"|", "rules":[
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"},
{"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"},