joe/OptiTrainer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add timeouts to prevent long DB locking.

Browse Source
This commit is contained in:
Joe Tretter
2024-08-20 16:22:39 -05:00
parent ba4df998fb
commit 12fd00b3cd
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/quoteDBs/continuousOptionTrainerLoad.sh
View File

@@ -3,9 +3,10 @@ while /bin/true; do
(set -e (set -e
find /home/schwab/OptionRecorderSchwab/data/ -type f -cnewer lastRun.stamp | grep -v -e '\..TM\.' | sort | while read singleFile; do find /home/schwab/OptionRecorderSchwab/data/ -type f -cnewer lastRun.stamp | grep -v -e '\..TM\.' | sort | while read singleFile; do
echo Processing ${singleFile} | tee | systemd-cat -t "`basename $0`" -p info echo Processing ${singleFile} | tee | systemd-cat -t "`basename $0`" -p info
symbol=$(pbzip2 -dc "${singleFile}" | jq -r '.symbol' | tr -d '$') symbol=$(pbzip2 -dc "${singleFile}" | jq -e -r '.symbol' | tr -d '$')
timeStamp=$(basename "${singleFile}" | cut -c 1-19 | tr '_' ":") timeStamp=$(basename "${singleFile}" | cut -c 1-19 | tr '_' ":")
insertSQL=$(cat <<EOF insertSQL=$(cat <<EOF
.timeout 10000;
insert into OptionQuotes ( insert into OptionQuotes (
TimeStamp, TimeStamp,
Symbol, Symbol,
@@ -27,7 +28,8 @@ EOF
set +e set +e
echo filling symbols table | tee | systemd-cat -t "`basename $0`" -p info echo filling symbols table | tee | systemd-cat -t "`basename $0`" -p info
sqlite3 consolidated.sqlite3 'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);' sqlite3 consolidated.sqlite3 '.timeout 10000'\\n'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
inotifywait -e create -r /home/schwab/OptionRecorderSchwab/data/ | tee | systemd-cat -t "`basename $0`" -p info inotifywait -e create -r /home/schwab/OptionRecorderSchwab/data/ | tee | systemd-cat -t "`basename $0`" -p info
sleep 200 # wait 200 seconds - this should be enough time for all files to finish writing... sleep 200 # wait 200 seconds - this should be enough time for all files to finish writing...