logAggregator/DBExtractor/DBExtractor.js

const fs = require("fs");
const config = JSON.parse(fs.readFileSync("DBExtractor.config.json", "utf8"));

const sql = require(config.dbDriver);
const request = require("request-promise");
const winston = require("winston");

const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.printf(info => {
            return `${info.timestamp} ${info.level}: ${info.message}`;
        })
    ),
    transports: [
        new winston.transports.Console(),
        new winston.transports.File({ filename: '../log/DBExtractorError.log', level: 'error' }),
        new winston.transports.File({ filename: '../log/Combined.log' })
    ]
});

console.log(`DBExtractor  - by j.tretter@gmail.com
    
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.`);

logger.log("info", `DBExtractor v0.1 Starting`);


logger.log("debug", "Config" + JSON.stringify(config))
let maxPKey = "";
let maxPKeyUnsent = "";
let allRows = [];

let theInterval;


try {
    lastOutMaxPKey = maxPKey = fs.readFileSync("DBExtractor.maxPKey.dat", "utf8");
    logger.log("debug", "MaxPKey Loaded: " + maxPKey);
} catch (ex) {
    if (ex.errno === -4058) {
			logger.log("info", "MaxPKey-File not existing. All data will be processed");
    } else {
        logger.log("error", "opening DBExtractor.maxPKey.dat :" + ex);
    }
}

try {
	sql.connect(config.sqlConfig, err => {
		if (err) {
			logger.log("error", "Connecting to DB: " + err);
		}
		const sqlRequest = new sql.Request();
		sqlRequest.stream = true;

		function runNextPoll(){
			sqlRequest.query(eval(config.statements.poller));
		}
		
		runNextPoll();

		sqlRequest.on('row', row => {
			outData = { sourceTag: config.sourceTag, processingType: config.processingType, data: row };
			process.stdout.write("#");
			request.json = true;

			logger.log('debug', "adding row to array:" + JSON.stringify(outData));
			allRows.push(outData);

			if (row.PKey > maxPKeyUnsent) {
				maxPKeyUnsent = row.PKey;
				logger.log("debug", "MaxPKeyUnsent Written: " + maxPKeyUnsent);
			}
		});

		sqlRequest.on('done', result => {
			clearInterval(theInterval);
			if (allRows.length>0){
				request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => {
					// count up the maxPKey only when the request was successful...
					maxPKey=maxPKeyUnsent;
					fs.writeFileSync("DBExtractor.maxPKey.dat", maxPKey);
				}).catch(err=>{
					logger.log("error","Sending Request to Service: " + err)
				}).finally(_=>{
					runNextPoll();
				});
				process.stdout.write(">" + allRows.length + ">");
			} else {
				if (config.pollMode) {
					theInterval = setInterval(runNextPoll, config.pollIntervalIdle);
					process.stdout.write(".");
				} else {
					sql.close();
					return(true);
				}
			}
			allRows=[];

		});
	})
} catch (ex) {
	logger.log("error", `Fatal error occured: ${ex}`);	
}