more improvements

RuleEngine.js

@@ -16,14 +16,35 @@ const logger = winston.createLogger({
     ]
 });
 
-
 exports.RuleEngine = class RuleEngine {
-    constructor(ruleSetFileMask) {
+    constructor(ruleSetFiles) {
-        this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFileMask + '.rule.json', 'utf8'));
+        logger.log("info", "Instanciated RulesEngine - RuleSetFiles: " + ruleSetFiles);
+        this.ruleSet = {};
+        if (ruleSetFiles === "*") {
+            fs.readdirSync("./rules/").forEach(file => {
+                if (this.reTest("/.*\.rule\.json$/", file)) {
+                    logger.log("info", "Loading ruleset file " + file);
+                    let newRuleSet = JSON.parse(fs.readFileSync('./rules/' + file, 'utf8'));
+                    logger.log("info", "Adding ruleset " + Object.keys(newRuleSet)[0]);
+                    this.ruleSet[Object.keys(newRuleSet)[0]] = newRuleSet[Object.keys(newRuleSet)[0]]
+                }
+            });
+        } else if (ruleSetFiles.indexOf(",") > 1) {
+            ruleSetFiles.split(",").forEach(file => {
+                logger.log("info", "Loading ruleset file " + file);
+                let newRuleSet = JSON.parse(fs.readFileSync('./rules/' + file + '.rule.json', 'utf8'));
+                logger.log("info", "Adding ruleset " + Object.keys(newRuleSet)[0]);
+                this.ruleSet[Object.keys(newRuleSet)[0]] = newRuleSet[Object.keys(newRuleSet)[0]]
+            });
+        } else {
+            logger.log("info", "Loading ruleset file " + ruleSetFiles);
+            this.ruleSet = JSON.parse(fs.readFileSync('./rules/' + ruleSetFiles + '.rule.json', 'utf8'));
+            logger.log("info", "Adding Ruleset " + Object.keys(this.ruleSet)[0]);
+        }
     }
     processData(inputLine) {
         let inputData = inputLine.data;
-        logger.log("debug", "ProcessingType " + inputLine.processingType + " RuleSet: " + this.ruleSet);
+        logger.log("debug", "ProcessingType " + inputLine.processingType);
         let allWhiteFilterRules = this.ruleSet[inputLine.processingType].rules;
         if (this.processLogicOr(allWhiteFilterRules, inputData)) { return (true); }
         logger.log("debug", "No whitelisting matched; returning false");
@@ -59,12 +80,12 @@ exports.RuleEngine = class RuleEngine {
         logger.log("debug", "Logic OR - start");
         for (let i = 0; i < allWhiteFilterRules.length; i++) {
             let theWhiteFilterRule = allWhiteFilterRules[i];
-            logger.log("debug", "  Logic OR - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule));
+            logger.log("debug", "  Logic OR - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule));
             if (theWhiteFilterRule.Logic === "-") {
                 if (this.processLogicFlat(theWhiteFilterRule, inputData)) {
                     logger.log("debug", "  Logic OR - FLAT - RETURNING OUT");
-                        return (true); 
+                    return (true);
-                    };
+                };
             } else if (theWhiteFilterRule.Logic === "|") {
                 if (this.processLogicOr(theWhiteFilterRule.rules, inputData)) {
                     logger.log("debug", "  Logic OR - OR - RETURNING OUT");
@@ -78,26 +99,42 @@ exports.RuleEngine = class RuleEngine {
             }
         };
         logger.log("debug", "Logic OR - end");
-        return(false);
+        return (false);
     }
     processLogicFlat(theWhiteFilterRule, inputData) {
-        logger.log("debug", "Logic FLAT start - whiteFilterRule: "+JSON.stringify(theWhiteFilterRule));
+        logger.log("debug", "Logic FLAT start - whiteFilterRule: " + JSON.stringify(theWhiteFilterRule));
+        let isKnownComp = false;
+        let retVal = false;
         if (theWhiteFilterRule.Comp === "===") {
-            if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) { 
+            isKnownComp = true;
-                logger.log("debug", "Logic FLAT - Exiting out");
+            if (inputData[theWhiteFilterRule.Field] === theWhiteFilterRule.Value) {retVal = true;}
-                return (true); 
+        } else if (theWhiteFilterRule.Comp === "<") {
-            }
+            isKnownComp = true;
+            if (inputData[theWhiteFilterRule.Field] < theWhiteFilterRule.Value) {retVal = true;}
+        } else if (theWhiteFilterRule.Comp === ">") {
+            isKnownComp = true;
+            if (inputData[theWhiteFilterRule.Field] > theWhiteFilterRule.Value) {retVal = true;}
+        } else if (theWhiteFilterRule.Comp === "<=") {
+            isKnownComp = true;
+            if (inputData[theWhiteFilterRule.Field] <= theWhiteFilterRule.Value) {retVal = true;}
+        } else if (theWhiteFilterRule.Comp === ">=") {
+            isKnownComp = true;
+            if (inputData[theWhiteFilterRule.Field] >= theWhiteFilterRule.Value) {retVal = true;}
         } else if (theWhiteFilterRule.Comp === "~") {
-            logger.log("debug","RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field]);
+            isKnownComp = true;
+            logger.log("debug", "RegexFilter: " + theWhiteFilterRule.Value + " || " + inputData[theWhiteFilterRule.Field].replace(/\r/g, "\\r").replace(/\n/g, "\\n"));
+            retVal = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]);
+        }
 
-            let result = this.reTest(theWhiteFilterRule.Value, inputData[theWhiteFilterRule.Field]);
+        if (!isKnownComp) {
+            logger.log("error", "The configured Comp '" + theWhiteFilterRule.Comp + "' is unknown.")
+        }
 
-            if (result) { 
+        if (retVal === true) {
-                logger.log("debug", "Logic FLAT - Exiting out");
+            logger.log("debug", "Logic FLAT - Exiting out");
-                return (true); 
-            }
         }
         logger.log("debug", "Logic FLAT - end");
+        return (retVal);
     }
 
     reTest(regex, data) {

casRpsLogExtractor/casRpsLogExtractor.js

@@ -60,6 +60,8 @@ sql.connect(config.sqlConfig, err => {
                 fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
                 logger.log("debug", "MaxPKey Written: " + maxPKey);
             }
+        }).catch(err=>{
+            logger.log("error","Sending Request to Service: " + err)
         });
     });
 

casRpsLogExtractor/casRpsLogExtractor.maxPKey.dat

@@ -1 +1 @@
-0010000000a87kiz
+0010000000a87kj9

logAggregator.js

@@ -21,26 +21,26 @@ const logger = winston.createLogger({
 });
 
 
-const ruleEngine = new RuleEngine("casrpslog");
+const ruleEngine = new RuleEngine("*");
-logger.log("info","Service Starting.");
+logger.log("info", "Service Starting.");
 const server = micro(async (req, res) => {
     const js = await json(req);
 
     try {
         inputLine = JSON.parse(js);
     } catch (ex) {
-        logger.log("error","Input not in JSON format:" + line);
+        logger.log("error", "Input not in JSON format:" + line);
     }
 
     let isWhiteListed = ruleEngine.processData(inputLine);
-    let miniInput=jsonminify(JSON.stringify(inputLine)) ;
+    let miniInput = jsonminify(JSON.stringify(inputLine));
     let now = Date();
     if (isWhiteListed) {
-        logger.log("debug","Not Whitelisted: " + miniInput);
+        logger.log("debug", "Not Whitelisted: " + miniInput);
-        fs.appendFileSync("./log/skipped.log",miniInput + "\n");
+        fs.appendFileSync("./log/skipped.log", miniInput + "\n");
     } else {
-        logger.log("info","Not Whitelisted: " + miniInput);
+        logger.log("info", "Not Whitelisted: " + miniInput);
-        fs.appendFileSync("./log/issues.log",miniInput + "\n");
+        fs.appendFileSync("./log/issues.log", miniInput + "\n");
     }
 
     send(res, 200, "OK");

rules/casroslogfull.rule.json

@@ -1,26 +0,0 @@
-{ "casrpslog":{ 
-    "rules":[
-        {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"},
-        {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"},
-        {"Logic":"&", "rules":[
-            {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"},
-            {"Logic":"|", "rules":[
-              {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"},
-              {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"}
-            ]}
-          ] 
-        },
-        {"Logic":"&", "rules":[
-          {"Logic":"|", "rules":[
-            {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"},
-            {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"}
-          ]},
-          {"Logic":"|", "rules":[
-            {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Integration web service started."},
-            {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"No setting value found in the database."},
-            {"Logic":"-", "Field":"LogText", "Comp":"===", "Value":"Starting Worker"}
-          ]}
-        ]}
-      ]
-    }
-  }

rules/casrpslog.rule.json

@@ -1,16 +1,16 @@
 { "casrpslog":{ 
   "rules":[
-      {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"Dummy"}, 
+      { "_comment":"Application Validation Errors",
-      {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*started.*/"},
+        "Logic":"&", "rules":[
-      {"Logic":"&", "rules":[
+          {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CAS 11.0"},
-          {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"CASApp"},
           {"Logic":"|", "rules":[
             {"Logic":"-", "Field":"Severity", "Comp":"<", "Value":"2"},
-            {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/.*Validation Notification.*/"}
+            {"Logic":"-", "Field":"LogText", "Comp":"~", "Value":"/^The promotion or tactic is not allowed to extend past the last day of the year.*/"}
           ]}
         ] 
       },
-      {"Logic":"&", "rules":[
+      {"_comment":"Worker Process info messages",
+        "Logic":"&", "rules":[
         {"Logic":"|", "rules":[
           {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"IntegrationWebService"},
           {"Logic":"-", "Field":"Module", "Comp":"===", "Value":"GenericBusinessLogicWorker"},