joe/RecordAppInsights
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Misc. changes

Browse Source
This commit is contained in:
Joe Tretter
2020-10-23 17:49:37 +01:00
parent a460398d27
commit 0ccc453326
4 changed files with 23 additions and 5008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
recordAppInsights.sh
View File

@@ -1,13 +1,23 @@
#!/bin/bash
# Application Insights JSON data Webservice and write into a SQL-Lite DB
# 2020-10-22 joe.tretter@kcc.com - Initial creation
cd `dirname "$0"`
mkdir -p inputdata
mkdir -p inputdata tmp
. ./config.sh
timestamp=$(date +%Y%m%d%H%M%S)
if [[ "$1" == "" ]]; then
filename=inputdata/exp_$(date +%Y%m%d%H%M%S).json
filename=inputdata/exp_${timestamp}.json
echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
echo "$(date +%Y%m%d%H%M%S): - Fetching JSON data"
curl -o ${filename} -H "X-Api-Key: nwh7mwsnawjzopnhqmhf15ha37x38dtmwihvdbbk" 'https://api.applicationinsights.io/v1/apps/acd05e9b-705f-4c89-8eb0-fc5035913ac7/events/requests?timespan=P7D&$top=500000'
# There is a hard 5000 record limit when using this API, can be lifted by switching to the Analytics API
# (https://dev.applicationinsights.io/documentation/Using-the-API/Query) which would have a different output format.
# For the current time, I believe we are not running 5K requests per day. And if we did, we can still run the script more often....
curl -o ${filename} -H "X-Api-Key: $ApiKey" "${Url}"
else
echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
echo "$(date +%Y%m%d%H%M%S): reprocess file"
@@ -15,10 +25,8 @@ else
fi
echo "$(date +%Y%m%d%H%M%S): - Processing JSON data into SQL Providerfile"
rm -f fillStage.sql
singleObjJSON=
declare -gA fields
objNestLevel=0
objNestLevel=0 # used to determine the end over ever Object "record" in the input JSON.
cat ${filename} | jq '.value[]' | while read x; do
# echo -n "_"$x"_"
if [[ "${x}" == *"{"* ]]; then
@@ -34,7 +42,7 @@ cat ${filename} | jq '.value[]' | while read x; do
singleObjJSON=${singleObjJSON}$x
if [[ 0 -eq $objNestLevel ]]; then
insertSQL="insert into stage (id,timestamp,name,success,resultCode,duration,performanceBucket,userid) values ($(echo ${singleObjJSON}|jq -r "\"'\(.request.id)','\(.timestamp)','\(.request.name)',\(.request.success),\(.request.resultCode),\(.request.duration),'\(.request.performanceBucket)','\(.user.authenticatedId)'\""));"
echo $insertSQL >> fillStage.sql
echo $insertSQL >> tmp/fillStage.${timestamp}.sql
echo -n "."
singleObjJSON=
fi
@@ -42,8 +50,8 @@ done
echo
echo "$(date +%Y%m%d%H%M%S): - Load into DB"
cat prepareStage.sql fillStage.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3
cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/
cat prepareStage.sql tmp/fillStage.${timestamp}.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3
#cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/
echo "$(date +%Y%m%d%H%M%S): END"