37 lines
1.4 KiB
Bash
37 lines
1.4 KiB
Bash
#!/bin/bash
|
|
while /bin/true; do
|
|
(set -e
|
|
find /home/schwab/OptionRecorderSchwab/data/ -type f -cnewer lastRun.stamp | grep -v -e '\..TM\.' | sort | while read singleFile; do
|
|
echo Processing ${singleFile} | tee | systemd-cat -t "`basename $0`" -p info
|
|
symbol=$(pbzip2 -dc "${singleFile}" | jq -e -r '.symbol' | tr -d '$')
|
|
timeStamp=$(basename "${singleFile}" | cut -c 1-19 | tr '_' ":")
|
|
insertSQL=$(cat <<EOF
|
|
.timeout 10000;
|
|
insert into OptionQuotes (
|
|
TimeStamp,
|
|
Symbol,
|
|
Data
|
|
) select
|
|
datetime('${timeStamp}','utc'),
|
|
'${symbol}',
|
|
'$(pbzip2 -dc "${singleFile}")'
|
|
where not exists (select 1
|
|
from OptionQuotes
|
|
where TimeStamp=datetime('${timeStamp}','utc')
|
|
and Symbol='${symbol}'
|
|
);
|
|
EOF
|
|
)
|
|
echo $insertSQL | sqlite3 consolidated.sqlite3
|
|
touch -r "${singleFile}" lastRun.stamp
|
|
done) 2>&1 | systemd-cat -t "`basename $0`" -p debug
|
|
|
|
set +e
|
|
echo filling symbols table | tee | systemd-cat -t "`basename $0`" -p info
|
|
sqlite3 consolidated.sqlite3 '.timeout 10000'\\n'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
|
|
|
|
|
|
inotifywait -e create -r /home/schwab/OptionRecorderSchwab/data/ | tee | systemd-cat -t "`basename $0`" -p info
|
|
sleep 200 # wait 200 seconds - this should be enough time for all files to finish writing...
|
|
done
|