logAggregator/casRpsLogExtractor/casRpsLogExtractor.js

const sql = require("mssql");
const request = require("request-promise");
const fs = require("fs");
const winston = require("winston");

const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.printf(info => {
            return `${info.timestamp} ${info.level}: ${info.message}`;
        })
    ),
    transports: [
        new winston.transports.Console(),
        new winston.transports.File({ filename: './log/casRpsLogExtractorError.log', level: 'error' }),
        new winston.transports.File({ filename: './log/Combined.log' })
    ]
});

logger.log("info", "Starting casRpsLogExtractor");
const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf8"));

logger.log("debug", "Config" + JSON.stringify(config))
const processingType = "casrpslog";
let maxPKey = "";
let lastOutMaxPKey = "";
try {
    lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8");
    logger.log("debug", "MaxPKey Loaded: " + maxPKey);
} catch (ex) {
    if (ex.errno === -4058) {
        logger.log("info", "MaxPKey-File not existing. Will be created.");
    } else {
        logger.log("error", "opening casRpsLogExtractor.maxPKey.dat :" + ex);
    }
}

sql.connect(config.sqlConfig, err => {
    if (err) {
        logger.log("error", "Connecting to DB: " + err);
    }
    const sqlRequest = new sql.Request();
    sqlRequest.stream = true;

    setInterval(_ => {
        sqlRequest.query("select top 100 * from rpslog where pkey > '" + maxPKey + "' order by pkey");
    }, config.pollInterval)

    sqlRequest.on('row', row => {
        outData = { sourceTag: config.sourceTag, processingType: processingType, data: row };
        process.stdout.write("#");
        request.json = true;

        logger.log('debug', "sending data:" + JSON.stringify(outData));
        request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(outData), json: true }).then(_ => {
            // count up the maxPKey only when the request was successful...
            if (row.PKey > maxPKey) {
                maxPKey = row.PKey;
                fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
                logger.log("debug", "MaxPKey Written: " + maxPKey);
            }
        }).catch(err=>{
            logger.log("error","Sending Request to Service: " + err)
        });
    });

    sqlRequest.on('done', result => {
        if (lastOutMaxPKey !== maxPKey) {
            process.stdout.write("[" + maxPKey + "]\n");
            lastOutMaxPKey = maxPKey;
        } else {
            process.stdout.write(".");
        }
    });
})