DB-processing, split initial and update

server/quoteDBs/fillBigDB_initial.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-rm consolidated.sqlite3
+mkdir -p processedDBs
 
+rm consolidated.sqlite3
 sqlite3 consolidated.sqlite3 < createTable.sql
 
 ls quoteStore_202*.sqlite3.gz | while read db; do 
@@ -11,4 +12,5 @@ ls quoteStore_202*.sqlite3.gz | while read db; do
 	sqlite3 /tmp/xx.sqlite3  '.dump OptionQuotes' | sqlite3 consolidated.sqlite3
 	echo deleting...
 	rm /tmp/xx.sqlite3
+	mv $db processedDBs 
 done

server/quoteDBs/fillBigDB_update.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+mkdir -p processedDBs
+
+ls quoteStore_202*.sqlite3.gz | while read db; do 
+	echo unpacking $db
+	gzip -dc $db >/tmp/xx.sqlite3
+	echo processing...
+	sqlite3 /tmp/xx.sqlite3  '.dump OptionQuotes' | sqlite3 consolidated.sqlite3
+	echo deleting...
+	rm /tmp/xx.sqlite3
+	mv $db processedDBs 
+done