#!/bin/bash

mkdir -p processedDBs

rm consolidated.sqlite3
sqlite3 consolidated.sqlite3 < createTable.sql

ls quoteStore_202*.sqlite3.gz | while read db; do 
	echo unpacking $db
	gzip -dc $db >/tmp/xx.sqlite3
	echo processing...
	sqlite3 /tmp/xx.sqlite3 "create table tmp as select timestamp,replace(replace(symbol,'$',''),'.X','') symbol,data from OptionQuotes;"
	sqlite3 /tmp/xx.sqlite3 'drop table OptionQuotes;'
	sqlite3 /tmp/xx.sqlite3 'alter table tmp rename to OptionQuotes;'
	sqlite3 /tmp/xx.sqlite3  '.dump OptionQuotes' | sqlite3 consolidated.sqlite3

	echo filling symbols table
	sqlite3 consolidated.sqlite3 'insert INTO Symbols select DISTINCT Symbol from OptionQuotes where OptionQuotes.Symbol not in (select Symbol from Symbols);'
	
	echo deleting...
	rm /tmp/xx.sqlite3
	mv $db processedDBs 
done