50 lines
1.6 KiB
Bash
50 lines
1.6 KiB
Bash
|
#!/bin/bash
|
||
|
|
cd `dirname "$0"`
|
||
|
|
mkdir -p inputdata
|
||
|
|
|
||
|
|
if [[ "$1" == "" ]]; then
|
||
|
|
filename=inputdata/exp_$(date +%Y%m%d%H%M%S).json
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
|
||
|
|
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): - Fetching JSON data"
|
||
|
|
curl -o ${filename} -H "X-Api-Key: nwh7mwsnawjzopnhqmhf15ha37x38dtmwihvdbbk" 'https://api.applicationinsights.io/v1/apps/acd05e9b-705f-4c89-8eb0-fc5035913ac7/events/requests?timespan=P7D&$top=500000'
|
||
|
|
else
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): START [${filename}]"
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): reprocess file"
|
||
|
|
filename=$1
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): - Processing JSON data into SQL Providerfile"
|
||
|
|
rm -f fillStage.sql
|
||
|
|
singleObjJSON=
|
||
|
|
declare -gA fields
|
||
|
|
objNestLevel=0
|
||
|
|
cat ${filename} | jq '.value[]' | while read x; do
|
||
|
|
# echo -n "_"$x"_"
|
||
|
|
if [[ "${x}" == *"{"* ]]; then
|
||
|
|
let objNestLevel=objNestLevel+1
|
||
|
|
# echo -n ">>>>>"
|
||
|
|
fi
|
||
|
|
if [[ "${x}" == *"}"* ]]; then
|
||
|
|
let objNestLevel=objNestLevel-1
|
||
|
|
# echo -n "<<<<<"
|
||
|
|
fi
|
||
|
|
# echo :${objNestLevel}:
|
||
|
|
|
||
|
|
singleObjJSON=${singleObjJSON}$x
|
||
|
|
if [[ 0 -eq $objNestLevel ]]; then
|
||
|
|
insertSQL="insert into stage (id,timestamp,name,success,resultCode,duration,performanceBucket,userid) values ($(echo ${singleObjJSON}|jq -r "\"'\(.request.id)','\(.timestamp)','\(.request.name)',\(.request.success),\(.request.resultCode),\(.request.duration),'\(.request.performanceBucket)','\(.user.authenticatedId)'\""));"
|
||
|
|
echo $insertSQL >> fillStage.sql
|
||
|
|
echo -n "."
|
||
|
|
singleObjJSON=
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): - Load into DB"
|
||
|
|
cat prepareStage.sql fillStage.sql stats.sql mergeData.sql stats.sql| sqlite3 db.sqlite3
|
||
|
|
cp db.sqlite3 //ustca239/JoesHop/TipsPerfDB/
|
||
|
|
|
||
|
|
echo "$(date +%Y%m%d%H%M%S): END"
|
||
|
|
|