diff --git a/.gitignore b/.gitignore index f1ac045..4729068 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,9 @@ .env -data/ \ No newline at end of file +data/ + +# Ignore Python bytecode files +*.pyc +__pycache__/ +*.pyo +*.pyd +*$py.class \ No newline at end of file diff --git a/backend/__pycache__/flex_lots.cpython-313.pyc b/backend/__pycache__/flex_lots.cpython-313.pyc index 7c5b30a..a7d699f 100644 Binary files a/backend/__pycache__/flex_lots.cpython-313.pyc and b/backend/__pycache__/flex_lots.cpython-313.pyc differ diff --git a/backend/__pycache__/flex_query.cpython-313.pyc b/backend/__pycache__/flex_query.cpython-313.pyc index 20b345c..21fce27 100644 Binary files a/backend/__pycache__/flex_query.cpython-313.pyc and b/backend/__pycache__/flex_query.cpython-313.pyc differ diff --git a/backend/__pycache__/ib_client.cpython-313.pyc b/backend/__pycache__/ib_client.cpython-313.pyc index 61b5bb1..864a4bc 100644 Binary files a/backend/__pycache__/ib_client.cpython-313.pyc and b/backend/__pycache__/ib_client.cpython-313.pyc differ diff --git a/backend/app.py b/backend/app.py index c29d6dc..624cc2f 100644 --- a/backend/app.py +++ b/backend/app.py @@ -16,6 +16,19 @@ from flex_query import FlexQueryError, fetch_flex_statement, get_flex_status from ib_client import IBClient import time +FLEX_AUTO_SYNC_MAX_AGE_SECONDS = 8 * 60 * 60 +_flex_sync_lock = threading.Lock() +_flex_startup_sync_started = False +_flex_sync_state = { + "last_attempt_at": None, + "last_success_at": None, + "last_result": None, + "last_error": None, + "last_saved_to": None, + "last_reference_code": None, + "last_trigger": None, +} + def load_dotenv_file() -> None: env_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".env")) @@ -41,18 +54,92 @@ load_dotenv_file() # Use an absolute path for the static folder so Flask can reliably serve files STATIC_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "frontend")) -app = Flask(__name__, static_folder=STATIC_FOLDER, static_url_path="/static") - -# instantiate the IB client with a process-unique client id to avoid -# collisions when Flask's reloader spawns multiple processes. -ib = IBClient(client_id=os.getpid()) +app = Flask(__name__, static_folder=STATIC_FOLDER, static_url_path="/static") + + +def should_auto_sync_flex(status: dict) -> bool: + if not status.get("configured"): + return False + latest_timestamp = status.get("latest_timestamp") + if latest_timestamp is None: + return True + return (time.time() - latest_timestamp) > FLEX_AUTO_SYNC_MAX_AGE_SECONDS + + +def get_flex_status_with_sync_state() -> dict: + status = get_flex_status() + status.update(_flex_sync_state) + return status + + +def run_flex_sync_if_needed(reason: str) -> bool: + status = get_flex_status() + if not should_auto_sync_flex(status): + return False + + if not _flex_sync_lock.acquire(blocking=False): + return False + + try: + _flex_sync_state["last_attempt_at"] = time.time() + _flex_sync_state["last_trigger"] = reason + _flex_sync_state["last_result"] = "running" + _flex_sync_state["last_error"] = None + + # Re-check after taking the lock so concurrent callers do not duplicate work. + status = get_flex_status() + if not should_auto_sync_flex(status): + _flex_sync_state["last_result"] = "skipped" + return False + result = fetch_flex_statement() + _flex_sync_state["last_success_at"] = time.time() + _flex_sync_state["last_result"] = "success" + _flex_sync_state["last_saved_to"] = result["saved_to"] + _flex_sync_state["last_reference_code"] = result["reference_code"] + print( + f"Flex sync completed ({reason}): saved_to={result['saved_to']} " + f"reference_code={result['reference_code']}" + ) + return True + except FlexQueryError as exc: + _flex_sync_state["last_result"] = "error" + _flex_sync_state["last_error"] = str(exc) + print(f"Flex sync skipped/failed ({reason}): {exc}") + return False + except Exception as exc: + _flex_sync_state["last_result"] = "error" + _flex_sync_state["last_error"] = str(exc) + print(f"Unexpected Flex sync failure ({reason}): {exc}") + return False + finally: + _flex_sync_lock.release() + + +def start_flex_startup_sync_if_needed() -> None: + global _flex_startup_sync_started + + if _flex_startup_sync_started: + return + _flex_startup_sync_started = True + + thread = threading.Thread( + target=run_flex_sync_if_needed, + args=("startup",), + daemon=True, + ) + thread.start() + +# instantiate the IB client with a process-unique client id to avoid +# collisions when Flask's reloader spawns multiple processes. +ib = IBClient(client_id=os.getpid()) # Start connection only in the main process (Werkzeug sets WERKZEUG_RUN_MAIN) if os.environ.get("WERKZEUG_RUN_MAIN") == "true" or not os.environ.get("WERKZEUG_RUN_MAIN"): # attempt to start the connection; failures are logged by the client - try: - ib.connect_and_start() - except Exception: - pass + try: + ib.connect_and_start() + except Exception: + pass + start_flex_startup_sync_if_needed() @app.route("/") @@ -216,7 +303,7 @@ def api_executions_debug(): @app.route("/api/flex/status") def api_flex_status(): - return jsonify(get_flex_status()) + return jsonify(get_flex_status_with_sync_state()) @app.route("/api/flex/lots-debug") @@ -236,11 +323,23 @@ def api_flex_lots_debug(): @app.route("/api/flex/sync", methods=["POST"]) def api_flex_sync(): try: + _flex_sync_state["last_attempt_at"] = time.time() + _flex_sync_state["last_trigger"] = "manual" + _flex_sync_state["last_result"] = "running" + _flex_sync_state["last_error"] = None result = fetch_flex_statement() + _flex_sync_state["last_success_at"] = time.time() + _flex_sync_state["last_result"] = "success" + _flex_sync_state["last_saved_to"] = result["saved_to"] + _flex_sync_state["last_reference_code"] = result["reference_code"] return jsonify({"ok": True, **result}) except FlexQueryError as exc: + _flex_sync_state["last_result"] = "error" + _flex_sync_state["last_error"] = str(exc) return jsonify({"ok": False, "error": str(exc)}), 400 except Exception as exc: + _flex_sync_state["last_result"] = "error" + _flex_sync_state["last_error"] = str(exc) return jsonify({"ok": False, "error": str(exc)}), 500 diff --git a/backend/flex_query.py b/backend/flex_query.py index 7dbafb1..6a40733 100644 --- a/backend/flex_query.py +++ b/backend/flex_query.py @@ -29,12 +29,18 @@ class FlexConfig: data_dir: Path poll_interval_seconds: float = 2.0 max_polls: int = 30 + send_request_retry_seconds: float = 10.0 + max_send_request_attempts: int = 6 class FlexQueryError(RuntimeError): pass +class FlexQueryRetryableError(FlexQueryError): + pass + + def load_flex_config() -> FlexConfig | None: token = os.environ.get("IB_FLEX_TOKEN", "").strip() query_id = os.environ.get("IB_FLEX_QUERY_ID", "").strip() @@ -86,7 +92,11 @@ def _parse_send_request(xml_bytes: bytes) -> tuple[str, str]: root = ET.fromstring(xml_bytes) status = (root.findtext("Status") or "").strip() if status.lower() != "success": - raise FlexQueryError(root.findtext("ErrorMessage") or "Flex send-request failed") + error_code = (root.findtext("ErrorCode") or "").strip() + error_message = (root.findtext("ErrorMessage") or "Flex send-request failed").strip() + if error_code == "1004": + raise FlexQueryRetryableError(error_message) + raise FlexQueryError(error_message) reference_code = (root.findtext("ReferenceCode") or "").strip() if not reference_code: raise FlexQueryError("Flex send-request returned no ReferenceCode") @@ -110,10 +120,23 @@ def fetch_flex_statement() -> dict: config.data_dir.mkdir(parents=True, exist_ok=True) - _, reference_code = _parse_send_request(_http_get( - FLEX_SEND_REQUEST_URL, - {"t": config.token, "q": config.query_id, "v": "3"}, - )) + reference_code = None + last_retryable_error = None + for attempt in range(1, config.max_send_request_attempts + 1): + try: + _, reference_code = _parse_send_request(_http_get( + FLEX_SEND_REQUEST_URL, + {"t": config.token, "q": config.query_id, "v": "3"}, + )) + break + except FlexQueryRetryableError as exc: + last_retryable_error = str(exc) + if attempt >= config.max_send_request_attempts: + raise FlexQueryError(last_retryable_error) from exc + time.sleep(config.send_request_retry_seconds) + + if not reference_code: + raise FlexQueryError(last_retryable_error or "Flex send-request returned no ReferenceCode") statement_xml = None for _ in range(config.max_polls):