commit 3addfb992e3c0b080af8f04df29d903bb9fe519d Author: Joe Tretter Date: Thu May 9 15:29:45 2024 -0500 Initial checkin diff --git a/README.md b/README.md new file mode 100644 index 0000000..09db7b4 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# Charls Schwab Quote recording bot + +## Expiry +* Access token expires in 30 Min. +* Refresh token expires in ? + +## File - Descriptions +### **`appCredentials.dat`** +* Base64 encoded App Credentials - `"{app key}:{secret}"` +* Manually maintained + +### **`appKey.dat`** +* only the application's AppKey +* Manually maintained + +#### **`allSymbols.json`** +* List of all Symbols to get Data for +* Manually maintained + +### **`access_token.dat`** +* Current Access Token +* written by `getNewAccessToken.sh` + +### **`accessTokenReqResp.json`** +* Full JSON response of the last Access Token request +* written by `getNewAccessToken.sh` + + ### **`refresh_token.dat`** +* Current refresh token +* written by `getNewRefreshToken.sh` + +### **`refreshTokenReqResp.json`** +* Full JSON response of the last Refresh Token request +* written by `getNewRefreshToken.sh` + +### **`codeReceiveServer.py`** +* Python HTTPS web server to receive and output the Authentication code from the Schwab Website callback +* This web server only runs temporarily and ends after receiving a request + +### **`cert.pem`** +* self signed certificate for the temporary HTTPS server that receives the code. +* Manually maintained with: + `openssl req -new -x509 -keyout cert.pem -out cert.pem -days 365 -nodes` + *Answer all questions hitting enter.* + +## **`data/`** +* Folder containing date sub-folders for the output data +* Created and maintained by `recordSingleSymbolSingleRun.sh` + +### **`getNewAccessToken.sh`** +* Gets a new access Token and saves it to file `access_token.dat` + +### **`getNewRefreshToken.sh`** +* Gets a new refresh Token and saves it to file `refresh_token.dat` + +### **`recordAllSymbolsSingleRun.sh`** +* Runs `recordSingleSymbolSingleRun.sh` for every Symbol in file `allSymbols.json` + +### **`recordQuoteSingleSymbolSingleRun.sh`** +* requests stock quote data for a single symbol and writes output into the `data/*` Folder structure. + +### **`recordOptionChainSingleSymbolSingleRun.sh`** +* requests option chain data for a single symbol and writes output into the `data/*` Folder structure. + + +# NEW CHANGES, NEED TO DOCUMENT +jq <<< '{ "numberOfContracts":'$(( $(jq '.numberOfContracts' < ITM.json) + $(jq '.numberOfContracts' < OTM.json) ))' }' > numberOfContracts.tmp +jq '. | del(.numberOfContracts) | map_values(select(type!="object"))' < ITM.json > CommonDetail.tmp +jq '. | map_values(select(type=="object"))' < ITM.json > ITMo.tmp +jq '. | map_values(select(type=="object"))' < OTM.json > OTMo.tmp +jq -s '.[0] * .[1] * .[2] * .[3]' numberOfContracts.tmp CommonDetail.tmp ITMo.tmp OTMo.tmp | less diff --git a/allSymbols.json.example b/allSymbols.json.example new file mode 100644 index 0000000..da99c47 --- /dev/null +++ b/allSymbols.json.example @@ -0,0 +1,101 @@ +[ + { + "Symbol": "AAPL" + }, + { + "Symbol": "ACN" + }, + { + "Symbol": "AMZN" + }, + { + "Symbol": "BAC" + }, + { + "Symbol": "CMG" + }, + { + "Symbol": "CRM" + }, + { + "Symbol": "CSCO" + }, + { + "Symbol": "DIS" + }, + { + "Symbol": "EDV" + }, + { + "Symbol": "EFA" + }, + { + "Symbol": "EWW" + }, + { + "Symbol": "FSLR" + }, + { + "Symbol": "GME" + }, + { + "Symbol": "GOOG" + }, + { + "Symbol": "HD" + }, + { + "Symbol": "INTC" + }, + { + "Symbol": "IWM" + }, + { + "Symbol": "KMB" + }, + { + "Symbol": "KO" + }, + { + "Symbol": "LVS" + }, + { + "Symbol": "MCD" + }, + { + "Symbol": "META" + }, + { + "Symbol": "NFLX" + }, + { + "Symbol": "PM" + }, + { + "Symbol": "PZZA" + }, + { + "Symbol": "QQQ" + }, + { + "Symbol": "SLV" + }, + { + "Symbol": "SPY" + }, + { + "Symbol": "TLT" + }, + { + "Symbol": "$VIX" + }, + { + "Symbol": "VXX" + }, + { + "Symbol": "WMT" + }, + { + "Symbol": "$XSP" + } +] diff --git a/codeReceiveServer.py b/codeReceiveServer.py new file mode 100755 index 0000000..a82ffe8 --- /dev/null +++ b/codeReceiveServer.py @@ -0,0 +1,30 @@ +#!/bin/python + +import http.server, ssl, socketserver +import urllib.parse + +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain("cert.pem") # PUT YOUR cert.pem HERE +server_address = ("0.0.0.0", 443) # CHANGE THIS IP & PORT +#handler = http.server.SimpleHTTPRequestHandler + +#class MyHandler(http.server.SimpleHTTPRequestHandler): +class MyHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + parsed_url = urllib.parse.urlparse(self.path) + code = urllib.parse.parse_qs(parsed_url.query).get('code', [None])[0] + if code: + print(code) + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write("OK.".encode("utf-8")) + self.wfile.flush() + self.wfile.close() + quit() + else: + print("Invalid or missing authorization code") + +with socketserver.TCPServer(server_address, MyHandler) as httpd: + httpd.socket = context.wrap_socket(httpd.socket, server_side=True) + httpd.serve_forever() diff --git a/getNewAccessToken.sh b/getNewAccessToken.sh new file mode 100755 index 0000000..51c44ad --- /dev/null +++ b/getNewAccessToken.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#set -x +curl -s -X POST https://api.schwabapi.com/v1/oauth/token \ + -H "Authorization: Basic $( temp/accessTokenReqResp.$$.json +jq -e -r .access_token temp/accessTokenReqResp.$$.json > access_token.dat +if [[ $? -ne 0 ]]; then + errMsg="ERROR: GETTING THE ACCESS TOKEN FAILED! $(&2 echo "${errMsg}" + curl -d "" -X POST "https://deadswitch.kawomi.com/api/v1?topic=trade.schwab.error.getNewAccessToken&secs=1&email=joe@kawomi.com" +fi +#echo Access Token: $( refresh_token.dat +#echo Refresh Token: $( temp/refreshTokenReqResp.$$.json +jq -r .refresh_token temp/refreshTokenReqResp.$$.json > refresh_token.dat +echo Refresh Token: $( temp/optionChain.${symbol}.tmp.$$.json + +jqQuery=".symbol" +symbolCheck=$(jq -r ${jqQuery} < temp/optionChain.${symbol}.tmp.$$.json) +if [[ "${symbol}" != "${symbolCheck}" ]]; then + errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/optionChain.${symbol}.tmp.$$.json_" + systemd-cat -t "$0 $1" -p err <<< ${errMsg} + >&2 echo "${errMsg}" + exit 2 +else + optionChainFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.json + #echo writing to file "${outDir}/${optionChainFileName}" + pbzip2 -9 -c < temp/optionChain.${symbol}.tmp.$$.json > "${outDir}/${optionChainFileName}.bz2" + rm temp/optionChain.${symbol}.tmp.$$.json +fi + + diff --git a/recordOptionChainSingleSymbolSpecialRun.sh b/recordOptionChainSingleSymbolSpecialRun.sh new file mode 100755 index 0000000..cb704fb --- /dev/null +++ b/recordOptionChainSingleSymbolSpecialRun.sh @@ -0,0 +1,43 @@ +#!/bin/bash +#set -x +# This is for big chains, that cause the Schwab API to error out (like $XSP) +# I add the range parameter (ITM/NTM/OTM) so that I get less data per request +# +symbol=$1 +outDir=data/$(date +%Y-%m-%d) + +mkdir -p "${outDir}" +mkdir -p temp + +for range in ITM OTM NTM; do + curl -s -X GET "https://api.schwabapi.com/marketdata/v1/chains?symbol=${symbol}&range=${range}" \ + -H "Authorization: Bearer $( temp/optionChain.${symbol}.${range}.tmp.$$.json + + jqQuery=".symbol" + symbolCheck=$(jq -r ${jqQuery} < temp/optionChain.${symbol}.${range}.tmp.$$.json) + if [[ "${symbol}" != "${symbolCheck}" ]]; then + errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/optionChain.${symbol}.${range}.tmp.$$.json_" + systemd-cat -t "$0 $1" -p err <<< ${errMsg} + >&2 echo "${errMsg}" + exit 2 + fi + + optionChainPartFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.${range}.json + pbzip2 -9 -c < temp/optionChain.${symbol}.${range}.tmp.$$.json > "${outDir}/${optionChainPartFileName}.bz2" +done + +###Paste together the parts, the NTM part, I don't think I need, but I will save it just in case.. +# sum up the number of Contracts + +ITMtempFile=temp/optionChain.${symbol}.ITM.tmp.$$.json +OTMtempFile=temp/optionChain.${symbol}.OTM.tmp.$$.json +jq <<< '{ "glued":true, "numberOfContracts":'$(( $(jq '.numberOfContracts' < ${ITMtempFile}) + $(jq '.numberOfContracts' < ${OTMtempFile}) ))' }' > temp/numberOfContracts.$$.tmp +jq '. | del(.numberOfContracts) | map_values(select(type!="object"))' < ${ITMtempFile} > temp/CommonDetail.$$.tmp +jq '. | map_values(select(type=="object"))' < ${ITMtempFile} > ${ITMtempFile}.objOnly +jq '. | map_values(select(type=="object"))' < ${OTMtempFile} > ${OTMtempFile}.objOnly +jq -s '.[0] * .[1] * .[2] * .[3]' temp/numberOfContracts.$$.tmp temp/CommonDetail.$$.tmp ${ITMtempFile}.objOnly ${OTMtempFile}.objOnly > temp/optionChain.${symbol}.tmp.$$.json + +optionChainFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.json +pbzip2 -9 -c < temp/optionChain.${symbol}.tmp.$$.json > "${outDir}/${optionChainFileName}.bz2" + +rm temp/*$$* diff --git a/recordQuoteSingleSymbolSingleRun.sh b/recordQuoteSingleSymbolSingleRun.sh new file mode 100755 index 0000000..da0bc5e --- /dev/null +++ b/recordQuoteSingleSymbolSingleRun.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#set -x +symbol=$1 +outDir=data/$(date +%Y-%m-%d) + +mkdir -p "${outDir}" +mkdir -p temp + +curl -s -X GET "https://api.schwabapi.com/marketdata/v1/${symbol}/quotes?fields=quote%2Creference" \ + -H "Authorization: Bearer $( temp/quote.${symbol}.tmp.$$.json + +jqQuery=".\"${symbol}\".symbol" +symbolCheck=$(jq -r ${jqQuery} < temp/quote.${symbol}.tmp.$$.json) +if [[ "${symbol}" != "${symbolCheck}" ]]; then + errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/quote.${symbol}.tmp.$$.json_" + systemd-cat -t "$0 $1" -p err <<< ${errMsg} + >&2 echo "${errMsg}" + exit 2 +else + quoteFileName=$(date "+%Y-%m-%d %H_%M_%S")_quote.${symbol}.json + #echo writing to file "${outDir}/${quoteFileName}" + pbzip2 -9 -c < temp/quote.${symbol}.tmp.$$.json > "${outDir}/${quoteFileName}.bz2" + rm temp/quote.${symbol}.tmp.$$.json +fi + +