From 3addfb992e3c0b080af8f04df29d903bb9fe519d Mon Sep 17 00:00:00 2001 From: Joe Tretter Date: Thu, 9 May 2024 15:29:45 -0500 Subject: [PATCH] Initial checkin --- README.md | 71 +++++++++++++++ allSymbols.json.example | 101 +++++++++++++++++++++ codeReceiveServer.py | 30 ++++++ getNewAccessToken.sh | 18 ++++ getNewRefreshToken.sh | 18 ++++ recordAllSymbolsSingleRun.sh | 35 +++++++ recordOptionChainSingleSymbolSingleRun.sh | 26 ++++++ recordOptionChainSingleSymbolSpecialRun.sh | 43 +++++++++ recordQuoteSingleSymbolSingleRun.sh | 26 ++++++ 9 files changed, 368 insertions(+) create mode 100644 README.md create mode 100644 allSymbols.json.example create mode 100755 codeReceiveServer.py create mode 100755 getNewAccessToken.sh create mode 100755 getNewRefreshToken.sh create mode 100755 recordAllSymbolsSingleRun.sh create mode 100755 recordOptionChainSingleSymbolSingleRun.sh create mode 100755 recordOptionChainSingleSymbolSpecialRun.sh create mode 100755 recordQuoteSingleSymbolSingleRun.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..09db7b4 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# Charls Schwab Quote recording bot + +## Expiry +* Access token expires in 30 Min. +* Refresh token expires in ? + +## File - Descriptions +### **`appCredentials.dat`** +* Base64 encoded App Credentials - `"{app key}:{secret}"` +* Manually maintained + +### **`appKey.dat`** +* only the application's AppKey +* Manually maintained + +#### **`allSymbols.json`** +* List of all Symbols to get Data for +* Manually maintained + +### **`access_token.dat`** +* Current Access Token +* written by `getNewAccessToken.sh` + +### **`accessTokenReqResp.json`** +* Full JSON response of the last Access Token request +* written by `getNewAccessToken.sh` + + ### **`refresh_token.dat`** +* Current refresh token +* written by `getNewRefreshToken.sh` + +### **`refreshTokenReqResp.json`** +* Full JSON response of the last Refresh Token request +* written by `getNewRefreshToken.sh` + +### **`codeReceiveServer.py`** +* Python HTTPS web server to receive and output the Authentication code from the Schwab Website callback +* This web server only runs temporarily and ends after receiving a request + +### **`cert.pem`** +* self signed certificate for the temporary HTTPS server that receives the code. +* Manually maintained with: + `openssl req -new -x509 -keyout cert.pem -out cert.pem -days 365 -nodes` + *Answer all questions hitting enter.* + +## **`data/`** +* Folder containing date sub-folders for the output data +* Created and maintained by `recordSingleSymbolSingleRun.sh` + +### **`getNewAccessToken.sh`** +* Gets a new access Token and saves it to file `access_token.dat` + +### **`getNewRefreshToken.sh`** +* Gets a new refresh Token and saves it to file `refresh_token.dat` + +### **`recordAllSymbolsSingleRun.sh`** +* Runs `recordSingleSymbolSingleRun.sh` for every Symbol in file `allSymbols.json` + +### **`recordQuoteSingleSymbolSingleRun.sh`** +* requests stock quote data for a single symbol and writes output into the `data/*` Folder structure. + +### **`recordOptionChainSingleSymbolSingleRun.sh`** +* requests option chain data for a single symbol and writes output into the `data/*` Folder structure. + + +# NEW CHANGES, NEED TO DOCUMENT +jq <<< '{ "numberOfContracts":'$(( $(jq '.numberOfContracts' < ITM.json) + $(jq '.numberOfContracts' < OTM.json) ))' }' > numberOfContracts.tmp +jq '. | del(.numberOfContracts) | map_values(select(type!="object"))' < ITM.json > CommonDetail.tmp +jq '. | map_values(select(type=="object"))' < ITM.json > ITMo.tmp +jq '. | map_values(select(type=="object"))' < OTM.json > OTMo.tmp +jq -s '.[0] * .[1] * .[2] * .[3]' numberOfContracts.tmp CommonDetail.tmp ITMo.tmp OTMo.tmp | less diff --git a/allSymbols.json.example b/allSymbols.json.example new file mode 100644 index 0000000..da99c47 --- /dev/null +++ b/allSymbols.json.example @@ -0,0 +1,101 @@ +[ + { + "Symbol": "AAPL" + }, + { + "Symbol": "ACN" + }, + { + "Symbol": "AMZN" + }, + { + "Symbol": "BAC" + }, + { + "Symbol": "CMG" + }, + { + "Symbol": "CRM" + }, + { + "Symbol": "CSCO" + }, + { + "Symbol": "DIS" + }, + { + "Symbol": "EDV" + }, + { + "Symbol": "EFA" + }, + { + "Symbol": "EWW" + }, + { + "Symbol": "FSLR" + }, + { + "Symbol": "GME" + }, + { + "Symbol": "GOOG" + }, + { + "Symbol": "HD" + }, + { + "Symbol": "INTC" + }, + { + "Symbol": "IWM" + }, + { + "Symbol": "KMB" + }, + { + "Symbol": "KO" + }, + { + "Symbol": "LVS" + }, + { + "Symbol": "MCD" + }, + { + "Symbol": "META" + }, + { + "Symbol": "NFLX" + }, + { + "Symbol": "PM" + }, + { + "Symbol": "PZZA" + }, + { + "Symbol": "QQQ" + }, + { + "Symbol": "SLV" + }, + { + "Symbol": "SPY" + }, + { + "Symbol": "TLT" + }, + { + "Symbol": "$VIX" + }, + { + "Symbol": "VXX" + }, + { + "Symbol": "WMT" + }, + { + "Symbol": "$XSP" + } +] diff --git a/codeReceiveServer.py b/codeReceiveServer.py new file mode 100755 index 0000000..a82ffe8 --- /dev/null +++ b/codeReceiveServer.py @@ -0,0 +1,30 @@ +#!/bin/python + +import http.server, ssl, socketserver +import urllib.parse + +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain("cert.pem") # PUT YOUR cert.pem HERE +server_address = ("0.0.0.0", 443) # CHANGE THIS IP & PORT +#handler = http.server.SimpleHTTPRequestHandler + +#class MyHandler(http.server.SimpleHTTPRequestHandler): +class MyHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + parsed_url = urllib.parse.urlparse(self.path) + code = urllib.parse.parse_qs(parsed_url.query).get('code', [None])[0] + if code: + print(code) + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write("OK.".encode("utf-8")) + self.wfile.flush() + self.wfile.close() + quit() + else: + print("Invalid or missing authorization code") + +with socketserver.TCPServer(server_address, MyHandler) as httpd: + httpd.socket = context.wrap_socket(httpd.socket, server_side=True) + httpd.serve_forever() diff --git a/getNewAccessToken.sh b/getNewAccessToken.sh new file mode 100755 index 0000000..51c44ad --- /dev/null +++ b/getNewAccessToken.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#set -x +curl -s -X POST https://api.schwabapi.com/v1/oauth/token \ + -H "Authorization: Basic $( temp/accessTokenReqResp.$$.json +jq -e -r .access_token temp/accessTokenReqResp.$$.json > access_token.dat +if [[ $? -ne 0 ]]; then + errMsg="ERROR: GETTING THE ACCESS TOKEN FAILED! $(&2 echo "${errMsg}" + curl -d "" -X POST "https://deadswitch.kawomi.com/api/v1?topic=trade.schwab.error.getNewAccessToken&secs=1&email=joe@kawomi.com" +fi +#echo Access Token: $( refresh_token.dat +#echo Refresh Token: $( temp/refreshTokenReqResp.$$.json +jq -r .refresh_token temp/refreshTokenReqResp.$$.json > refresh_token.dat +echo Refresh Token: $( temp/optionChain.${symbol}.tmp.$$.json + +jqQuery=".symbol" +symbolCheck=$(jq -r ${jqQuery} < temp/optionChain.${symbol}.tmp.$$.json) +if [[ "${symbol}" != "${symbolCheck}" ]]; then + errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/optionChain.${symbol}.tmp.$$.json_" + systemd-cat -t "$0 $1" -p err <<< ${errMsg} + >&2 echo "${errMsg}" + exit 2 +else + optionChainFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.json + #echo writing to file "${outDir}/${optionChainFileName}" + pbzip2 -9 -c < temp/optionChain.${symbol}.tmp.$$.json > "${outDir}/${optionChainFileName}.bz2" + rm temp/optionChain.${symbol}.tmp.$$.json +fi + + diff --git a/recordOptionChainSingleSymbolSpecialRun.sh b/recordOptionChainSingleSymbolSpecialRun.sh new file mode 100755 index 0000000..cb704fb --- /dev/null +++ b/recordOptionChainSingleSymbolSpecialRun.sh @@ -0,0 +1,43 @@ +#!/bin/bash +#set -x +# This is for big chains, that cause the Schwab API to error out (like $XSP) +# I add the range parameter (ITM/NTM/OTM) so that I get less data per request +# +symbol=$1 +outDir=data/$(date +%Y-%m-%d) + +mkdir -p "${outDir}" +mkdir -p temp + +for range in ITM OTM NTM; do + curl -s -X GET "https://api.schwabapi.com/marketdata/v1/chains?symbol=${symbol}&range=${range}" \ + -H "Authorization: Bearer $( temp/optionChain.${symbol}.${range}.tmp.$$.json + + jqQuery=".symbol" + symbolCheck=$(jq -r ${jqQuery} < temp/optionChain.${symbol}.${range}.tmp.$$.json) + if [[ "${symbol}" != "${symbolCheck}" ]]; then + errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/optionChain.${symbol}.${range}.tmp.$$.json_" + systemd-cat -t "$0 $1" -p err <<< ${errMsg} + >&2 echo "${errMsg}" + exit 2 + fi + + optionChainPartFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.${range}.json + pbzip2 -9 -c < temp/optionChain.${symbol}.${range}.tmp.$$.json > "${outDir}/${optionChainPartFileName}.bz2" +done + +###Paste together the parts, the NTM part, I don't think I need, but I will save it just in case.. +# sum up the number of Contracts + +ITMtempFile=temp/optionChain.${symbol}.ITM.tmp.$$.json +OTMtempFile=temp/optionChain.${symbol}.OTM.tmp.$$.json +jq <<< '{ "glued":true, "numberOfContracts":'$(( $(jq '.numberOfContracts' < ${ITMtempFile}) + $(jq '.numberOfContracts' < ${OTMtempFile}) ))' }' > temp/numberOfContracts.$$.tmp +jq '. | del(.numberOfContracts) | map_values(select(type!="object"))' < ${ITMtempFile} > temp/CommonDetail.$$.tmp +jq '. | map_values(select(type=="object"))' < ${ITMtempFile} > ${ITMtempFile}.objOnly +jq '. | map_values(select(type=="object"))' < ${OTMtempFile} > ${OTMtempFile}.objOnly +jq -s '.[0] * .[1] * .[2] * .[3]' temp/numberOfContracts.$$.tmp temp/CommonDetail.$$.tmp ${ITMtempFile}.objOnly ${OTMtempFile}.objOnly > temp/optionChain.${symbol}.tmp.$$.json + +optionChainFileName=$(date "+%Y-%m-%d %H_%M_%S")_optionChain.${symbol}.json +pbzip2 -9 -c < temp/optionChain.${symbol}.tmp.$$.json > "${outDir}/${optionChainFileName}.bz2" + +rm temp/*$$* diff --git a/recordQuoteSingleSymbolSingleRun.sh b/recordQuoteSingleSymbolSingleRun.sh new file mode 100755 index 0000000..da0bc5e --- /dev/null +++ b/recordQuoteSingleSymbolSingleRun.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#set -x +symbol=$1 +outDir=data/$(date +%Y-%m-%d) + +mkdir -p "${outDir}" +mkdir -p temp + +curl -s -X GET "https://api.schwabapi.com/marketdata/v1/${symbol}/quotes?fields=quote%2Creference" \ + -H "Authorization: Bearer $( temp/quote.${symbol}.tmp.$$.json + +jqQuery=".\"${symbol}\".symbol" +symbolCheck=$(jq -r ${jqQuery} < temp/quote.${symbol}.tmp.$$.json) +if [[ "${symbol}" != "${symbolCheck}" ]]; then + errMsg="ERROR: THE RESPONSE IS INCORRECT! Symbol: _${symbol}_ Check: _${symbolCheck}_ check file _temp/quote.${symbol}.tmp.$$.json_" + systemd-cat -t "$0 $1" -p err <<< ${errMsg} + >&2 echo "${errMsg}" + exit 2 +else + quoteFileName=$(date "+%Y-%m-%d %H_%M_%S")_quote.${symbol}.json + #echo writing to file "${outDir}/${quoteFileName}" + pbzip2 -9 -c < temp/quote.${symbol}.tmp.$$.json > "${outDir}/${quoteFileName}.bz2" + rm temp/quote.${symbol}.tmp.$$.json +fi + +