From 12fd00b3cdbb491475cd85ece599fc87ab353071 Mon Sep 17 00:00:00 2001
From: Joe Tretter <joerg.tretter@gmail.com>
Date: Tue, 20 Aug 2024 16:22:39 -0500
Subject: [PATCH] add timeouts to prevent long DB locking.

---
 server/quoteDBs/continuousOptionTrainerLoad.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/server/quoteDBs/continuousOptionTrainerLoad.sh b/server/quoteDBs/continuousOptionTrainerLoad.sh
index 1310cd5..d7127ce 100644
--- a/server/quoteDBs/continuousOptionTrainerLoad.sh
+++ b/server/quoteDBs/continuousOptionTrainerLoad.sh
@@ -3,9 +3,10 @@ while /bin/true; do
   (set -e
   find /home/schwab/OptionRecorderSchwab/data/ -type f -cnewer lastRun.stamp | grep -v -e '\..TM\.' | sort | while read singleFile; do
     echo Processing ${singleFile} | tee | systemd-cat -t "`basename $0`" -p info 
-    symbol=$(pbzip2 -dc "${singleFile}" | jq -r '.symbol' | tr -d '$')
+    symbol=$(pbzip2 -dc "${singleFile}" | jq -e -r '.symbol' | tr -d '$')
     timeStamp=$(basename "${singleFile}" | cut -c 1-19 | tr '_' ":")
     insertSQL=$(cat <<EOF
+      .timeout 10000;
       insert into OptionQuotes (
         TimeStamp,
         Symbol,
@@ -27,7 +28,8 @@ EOF
   
   set +e
   echo filling symbols table | tee | systemd-cat -t "`basename $0`" -p info 
-  sqlite3 consolidated.sqlite3 'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
+  sqlite3 consolidated.sqlite3 '.timeout 10000'\\n'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
+      
   
   inotifywait -e create -r /home/schwab/OptionRecorderSchwab/data/ | tee | systemd-cat -t "`basename $0`" -p info 
   sleep 200 # wait 200 seconds - this should be enough time for all files to finish writing...