110 lines
3.7 KiB
JavaScript
110 lines
3.7 KiB
JavaScript
const sql = require("mssql/msnodesqlv8");
|
|
//const sql = require("mssql");
|
|
const request = require("request-promise");
|
|
const fs = require("fs");
|
|
const winston = require("winston");
|
|
|
|
const logger = winston.createLogger({
|
|
level: 'info',
|
|
format: winston.format.combine(
|
|
winston.format.timestamp(),
|
|
winston.format.printf(info => {
|
|
return `${info.timestamp} ${info.level}: ${info.message}`;
|
|
})
|
|
),
|
|
transports: [
|
|
new winston.transports.Console(),
|
|
new winston.transports.File({ filename: './log/casRpsLogExtractorError.log', level: 'error' }),
|
|
new winston.transports.File({ filename: './log/Combined.log' })
|
|
]
|
|
});
|
|
|
|
console.log(`casRpsLogExtractor Copyright (C) 2019 j.tretter@gmail.com
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <https://www.gnu.org/licenses/>.`);
|
|
|
|
logger.log("info", `casRpsLogExtractor v0.1 Starting`);
|
|
|
|
const config = JSON.parse(fs.readFileSync("casRpsLogExtractor.config.json", "utf8"));
|
|
|
|
logger.log("debug", "Config" + JSON.stringify(config))
|
|
const processingType = "casrpslog";
|
|
const maxResultsPerMessage=100;
|
|
let maxPKey = "";
|
|
let maxPKeyUnsent = "";
|
|
let allRows = [];
|
|
|
|
let theInterval;
|
|
|
|
|
|
try {
|
|
lastOutMaxPKey = maxPKey = fs.readFileSync("casRpsLogExtractor.maxPKey.dat", "utf8");
|
|
logger.log("debug", "MaxPKey Loaded: " + maxPKey);
|
|
} catch (ex) {
|
|
if (ex.errno === -4058) {
|
|
logger.log("info", "MaxPKey-File not existing. Will be created.");
|
|
} else {
|
|
logger.log("error", "opening casRpsLogExtractor.maxPKey.dat :" + ex);
|
|
}
|
|
}
|
|
|
|
sql.connect(config.sqlConfig, err => {
|
|
if (err) {
|
|
logger.log("error", "Connecting to DB: " + err);
|
|
}
|
|
const sqlRequest = new sql.Request();
|
|
sqlRequest.stream = true;
|
|
|
|
function runNextPoll(){
|
|
sqlRequest.query("select top " + maxResultsPerMessage + " * from rpslog where pkey > '" + maxPKey + "' order by pkey");
|
|
}
|
|
|
|
runNextPoll();
|
|
|
|
sqlRequest.on('row', row => {
|
|
outData = { sourceTag: config.sourceTag, processingType: processingType, data: row };
|
|
process.stdout.write("#");
|
|
request.json = true;
|
|
|
|
logger.log('debug', "adding row to array:" + JSON.stringify(outData));
|
|
allRows.push(outData);
|
|
|
|
if (row.PKey > maxPKeyUnsent) {
|
|
maxPKeyUnsent = row.PKey;
|
|
logger.log("debug", "MaxPKeyUnsent Written: " + maxPKeyUnsent);
|
|
}
|
|
});
|
|
|
|
sqlRequest.on('done', result => {
|
|
clearInterval(theInterval);
|
|
if (allRows.length>0){
|
|
request({ method: 'POST', uri: config.serverURI, body: JSON.stringify(allRows), json: true }).then(_ => {
|
|
// count up the maxPKey only when the request was successful...
|
|
maxPKey=maxPKeyUnsent;
|
|
fs.writeFileSync("casRpsLogExtractor.maxPKey.dat", maxPKey);
|
|
}).catch(err=>{
|
|
logger.log("error","Sending Request to Service: " + err)
|
|
}).finally(_=>{
|
|
runNextPoll();
|
|
});
|
|
process.stdout.write(">" + allRows.length + ">");
|
|
} else {
|
|
theInterval = setInterval(runNextPoll, config.pollIntervalIdle);
|
|
process.stdout.write(".");
|
|
}
|
|
allRows=[];
|
|
|
|
});
|
|
})
|