Generally quite nicely working already

backend/app.py

@@ -7,16 +7,40 @@ Provides simple REST endpoints that the vanilla-JS frontend polls:
 
 This app expects an IB Gateway or TWS running locally on port 7496.
 """
-from flask import Flask, jsonify, request, send_from_directory
-import os
-import threading
-
-from ib_client import IBClient
-import time
-
-
-# Use an absolute path for the static folder so Flask can reliably serve files
-STATIC_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "frontend"))
+from flask import Flask, jsonify, request, send_from_directory
+import os
+import threading
+
+from flex_lots import load_latest_flex_lots
+from flex_query import FlexQueryError, fetch_flex_statement, get_flex_status
+from ib_client import IBClient
+import time
+
+
+def load_dotenv_file() -> None:
+    env_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".env"))
+    if not os.path.exists(env_path):
+        return
+    try:
+        with open(env_path, "r", encoding="utf-8") as handle:
+            for raw_line in handle:
+                line = raw_line.strip()
+                if not line or line.startswith("#") or "=" not in line:
+                    continue
+                key, value = line.split("=", 1)
+                key = key.strip()
+                value = value.strip().strip('"').strip("'")
+                if key and key not in os.environ:
+                    os.environ[key] = value
+    except Exception:
+        pass
+
+
+load_dotenv_file()
+
+
+# Use an absolute path for the static folder so Flask can reliably serve files
+STATIC_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "frontend"))
 app = Flask(__name__, static_folder=STATIC_FOLDER, static_url_path="/static")
 
 # instantiate the IB client with a process-unique client id to avoid
@@ -59,6 +83,167 @@ def api_positions():
     return jsonify(ib.get_positions())
 
 
+@app.route("/api/lots")
+def api_lots():
+    flex_status = get_flex_status()
+    if flex_status.get("configured") and flex_status.get("data_dir"):
+        latest_file, flex_lots = load_latest_flex_lots(flex_status["data_dir"])
+        if latest_file and flex_lots:
+            return jsonify(flex_lots)
+
+    # request a fresh positions snapshot first so all account holdings are covered,
+    # even when IB does not return historical executions for every open position.
+    try:
+        ib.request_positions()
+    except Exception:
+        pass
+    positions_deadline = time.time() + 2.0
+    while time.time() < positions_deadline:
+        if ib.get_positions():
+            break
+        time.sleep(0.1)
+
+    for p in ib.get_positions():
+        account = p.get("account") if isinstance(p, dict) else None
+        if account:
+            try:
+                ib.request_account_updates(account)
+            except Exception:
+                pass
+
+    # subscribe market data for all current positions to improve gain calculations
+    for p in ib.get_positions():
+        instrument_key = p.get("instrument_key") if isinstance(p, dict) else None
+        if instrument_key:
+            try:
+                ib.subscribe_market_data(p)
+            except Exception:
+                pass
+    quotes_deadline = time.time() + 2.0
+    position_symbols = {
+        p.get("instrument_key")
+        for p in ib.get_positions()
+        if isinstance(p, dict) and p.get("instrument_key")
+    }
+    while time.time() < quotes_deadline:
+        quotes = ib.get_quotes()
+        if any(instrument_key in quotes for instrument_key in position_symbols):
+            break
+        time.sleep(0.1)
+
+    # request prior close for positions still missing a usable quote
+    for p in ib.get_positions():
+        instrument_key = p.get("instrument_key") if isinstance(p, dict) else None
+        if not instrument_key:
+            continue
+        quote = ib.get_quotes().get(instrument_key, {})
+        if quote.get("last") is None and quote.get("close") is None and quote.get("mark") is None:
+            try:
+                ib.request_historical_close(p)
+            except Exception:
+                pass
+    historical_deadline = time.time() + 2.0
+    while time.time() < historical_deadline:
+        missing = []
+        quotes = ib.get_quotes()
+        for p in ib.get_positions():
+            instrument_key = p.get("instrument_key") if isinstance(p, dict) else None
+            if not instrument_key:
+                continue
+            quote = quotes.get(instrument_key, {})
+            if (
+                quote.get("last") is None
+                and quote.get("close") is None
+                and quote.get("mark") is None
+                and instrument_key not in ib.historical_closes
+            ):
+                missing.append(instrument_key)
+        if not missing:
+            break
+        time.sleep(0.1)
+
+    # request executions and wait briefly for callbacks
+    try:
+        ib.request_executions(start_time="20000101 00:00:00")
+    except Exception:
+        pass
+    deadline = time.time() + 3.0
+    while time.time() < deadline:
+        lots = ib.compute_open_lots(include_position_fallback=False)
+        if lots:
+            return jsonify(lots)
+        time.sleep(0.1)
+    return jsonify(ib.compute_open_lots(include_position_fallback=False))
+
+
+@app.route("/api/executions-debug")
+def api_executions_debug():
+    start_time = request.args.get("start_time", "20000101 00:00:00")
+    try:
+        ib.request_executions(start_time=start_time)
+    except Exception:
+        pass
+    deadline = time.time() + 5.0
+    while time.time() < deadline:
+        time.sleep(0.1)
+    executions = list(ib.executions)
+    summary = {}
+    for e in executions:
+        key = e.get("instrument_key") or e.get("symbol") or "UNKNOWN"
+        item = summary.setdefault(key, {
+            "instrument_key": e.get("instrument_key"),
+            "symbol": e.get("symbol"),
+            "security_type": e.get("security_type"),
+            "display_symbol": e.get("display_symbol"),
+            "count": 0,
+            "first_time": None,
+            "last_time": None,
+        })
+        item["count"] += 1
+        etime = e.get("time")
+        if etime:
+            if item["first_time"] is None or etime < item["first_time"]:
+                item["first_time"] = etime
+            if item["last_time"] is None or etime > item["last_time"]:
+                item["last_time"] = etime
+    return jsonify({
+        "start_time": start_time,
+        "execution_count": len(executions),
+        "executions": executions,
+        "summary": list(summary.values()),
+    })
+
+
+@app.route("/api/flex/status")
+def api_flex_status():
+    return jsonify(get_flex_status())
+
+
+@app.route("/api/flex/lots-debug")
+def api_flex_lots_debug():
+    status = get_flex_status()
+    if not status.get("configured") or not status.get("data_dir"):
+        return jsonify({"configured": False, "lots": [], "latest_file": None})
+    latest_file, lots = load_latest_flex_lots(status["data_dir"])
+    return jsonify({
+        "configured": True,
+        "latest_file": latest_file,
+        "lot_count": len(lots),
+        "lots": lots,
+    })
+
+
+@app.route("/api/flex/sync", methods=["POST"])
+def api_flex_sync():
+    try:
+        result = fetch_flex_statement()
+        return jsonify({"ok": True, **result})
+    except FlexQueryError as exc:
+        return jsonify({"ok": False, "error": str(exc)}), 400
+    except Exception as exc:
+        return jsonify({"ok": False, "error": str(exc)}), 500
+
+
 @app.route("/api/subscribe", methods=["POST"])
 def api_subscribe():
     data = request.get_json(force=True)

backend/flex_lots.py

@@ -0,0 +1,144 @@
+"""Parse lot-level open positions from IB Flex XML files."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+
+def _as_float(value, default=None):
+    try:
+        if value in (None, ""):
+            return default
+        return float(value)
+    except Exception:
+        return default
+
+
+def _normalize_expiry(expiry: str | None) -> str | None:
+    expiry = (expiry or "").strip()
+    if len(expiry) == 8 and expiry.isdigit():
+        return f"{expiry[0:4]}-{expiry[4:6]}-{expiry[6:8]}"
+    return expiry or None
+
+
+def _normalize_buy_time(open_dt: str | None) -> str | None:
+    open_dt = (open_dt or "").strip()
+    if not open_dt or ";" not in open_dt:
+        return open_dt or None
+    date_part, time_part = open_dt.split(";", 1)
+    if len(date_part) == 8 and len(time_part) == 6:
+        return f"{date_part[0:4]}-{date_part[4:6]}-{date_part[6:8]} {time_part[0:2]}:{time_part[2:4]}:{time_part[4:6]}"
+    return open_dt
+
+
+def _compute_cagr_percent(cost_basis, gain_dollars, buy_time: str | None):
+    if cost_basis in (None, 0) or gain_dollars is None or not buy_time:
+        return None
+    try:
+        start = datetime.fromisoformat(buy_time).replace(tzinfo=timezone.utc)
+        now = datetime.now(timezone.utc)
+        years = max((now - start).total_seconds() / (365.25 * 24 * 3600), 1e-9)
+        start_value = abs(float(cost_basis))
+        end_value = start_value + float(gain_dollars)
+        if start_value <= 0 or end_value <= 0:
+            return None
+        return ((end_value / start_value) ** (1.0 / years) - 1.0) * 100.0
+    except Exception:
+        return None
+
+
+def _compute_days_since_buy(buy_time: str | None):
+    if not buy_time:
+        return None
+    try:
+        start = datetime.fromisoformat(buy_time).replace(tzinfo=timezone.utc)
+        now = datetime.now(timezone.utc)
+        return max((now - start).days, 0)
+    except Exception:
+        return None
+
+
+def _display_symbol(attrs: dict) -> str:
+    description = (attrs.get("description") or "").strip()
+    if description:
+        return description
+    return (attrs.get("underlyingSymbol") or attrs.get("symbol") or "").strip()
+
+
+def _instrument_key(attrs: dict) -> str:
+    conid = (attrs.get("conid") or "").strip()
+    if conid:
+        return f"CONID:{conid}"
+    return (attrs.get("symbol") or "").strip()
+
+
+def parse_flex_lots(xml_path: str | Path) -> list[dict]:
+    root = ET.parse(xml_path).getroot()
+    lots: list[dict] = []
+    for node in root.findall(".//OpenPosition"):
+        attrs = node.attrib
+        if (attrs.get("levelOfDetail") or "").upper() != "LOT":
+            continue
+
+        security_type = (attrs.get("assetCategory") or "").upper() or "STK"
+        multiplier = _as_float(attrs.get("multiplier"), 1.0) or 1.0
+        raw_position = _as_float(attrs.get("position"), 0.0) or 0.0
+        side = (attrs.get("side") or "").strip().lower()
+        qty = -abs(raw_position) if side == "short" or raw_position < 0 else abs(raw_position)
+
+        mark_price = _as_float(attrs.get("markPrice"))
+        market_price = mark_price
+        market_value = _as_float(attrs.get("positionValue"))
+        cost_basis = _as_float(attrs.get("costBasisMoney"))
+        avg_price = _as_float(attrs.get("costBasisPrice"))
+        if avg_price is None:
+            open_price = _as_float(attrs.get("openPrice"))
+            avg_price = open_price
+        if avg_price is None and qty:
+            divisor = qty * multiplier if security_type == "OPT" else qty
+            avg_price = abs(cost_basis / divisor) if cost_basis is not None and divisor else None
+        buy_time = _normalize_buy_time(attrs.get("openDateTime"))
+        gain_dollars = _as_float(attrs.get("fifoPnlUnrealized"))
+        gain_percent = None
+        if cost_basis not in (None, 0) and gain_dollars is not None:
+            gain_percent = (gain_dollars / abs(cost_basis)) * 100.0
+        cagr_percent = _compute_cagr_percent(cost_basis, gain_dollars, buy_time)
+        days_since_buy = _compute_days_since_buy(buy_time)
+        ltcg = "X" if days_since_buy is not None and days_since_buy >= 365 else ""
+
+        lots.append({
+            "instrument_key": _instrument_key(attrs),
+            "con_id": _as_float(attrs.get("conid")),
+            "symbol": attrs.get("symbol"),
+            "display_symbol": _display_symbol(attrs),
+            "security_type": security_type,
+            "underlying_symbol": attrs.get("underlyingSymbol") or attrs.get("symbol"),
+            "expiry": _normalize_expiry(attrs.get("expiry")),
+            "strike": _as_float(attrs.get("strike")),
+            "option_type": (attrs.get("putCall") or "").upper() or None,
+            "multiplier": str(int(multiplier)) if float(multiplier).is_integer() else str(multiplier),
+            "qty": qty,
+            "avg_price": avg_price,
+            "cost_basis": cost_basis,
+            "market_price": market_price,
+            "market_value": market_value,
+            "gain_dollars": gain_dollars,
+            "gain_percent": gain_percent,
+            "cagr_percent": cagr_percent,
+            "days_since_buy": days_since_buy,
+            "ltcg": ltcg,
+            "buy_time": buy_time,
+        })
+    return lots
+
+
+def load_latest_flex_lots(data_dir: str | Path) -> tuple[str | None, list[dict]]:
+    data_path = Path(data_dir)
+    if not data_path.exists():
+        return None, []
+    xml_files = sorted(data_path.glob("flex_*.xml"))
+    if not xml_files:
+        return None, []
+    latest = xml_files[-1]
+    return str(latest), parse_flex_lots(latest)

backend/flex_query.py

@@ -0,0 +1,141 @@
+"""Utilities for Interactive Brokers Flex Web Service integration.
+
+This module provides the first layer of infrastructure for pulling
+historical trade data from a saved Flex Query. It intentionally keeps the
+surface area small so the app can validate configuration, trigger a sync,
+and persist the raw XML responses locally before the data is wired into the
+lot engine.
+"""
+from __future__ import annotations
+
+import os
+import time
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from pathlib import Path
+
+
+FLEX_BASE_URL = "https://ndcdyn.interactivebrokers.com/AccountManagement/FlexWebService"
+FLEX_SEND_REQUEST_URL = f"{FLEX_BASE_URL}/SendRequest"
+FLEX_GET_STATEMENT_URL = f"{FLEX_BASE_URL}/GetStatement"
+
+
+@dataclass
+class FlexConfig:
+    token: str
+    query_id: str
+    data_dir: Path
+    poll_interval_seconds: float = 2.0
+    max_polls: int = 30
+
+
+class FlexQueryError(RuntimeError):
+    pass
+
+
+def load_flex_config() -> FlexConfig | None:
+    token = os.environ.get("IB_FLEX_TOKEN", "").strip()
+    query_id = os.environ.get("IB_FLEX_QUERY_ID", "").strip()
+    if not token or not query_id:
+        return None
+    data_dir = Path(os.environ.get("IB_FLEX_DATA_DIR", Path(__file__).resolve().parent.parent / "data" / "flex"))
+    return FlexConfig(token=token, query_id=query_id, data_dir=data_dir)
+
+
+def get_flex_status() -> dict:
+    config = load_flex_config()
+    if config is None:
+        return {
+            "configured": False,
+            "missing": [
+                name
+                for name in ("IB_FLEX_TOKEN", "IB_FLEX_QUERY_ID")
+                if not os.environ.get(name, "").strip()
+            ],
+        }
+    latest_file = None
+    latest_mtime = None
+    if config.data_dir.exists():
+        xml_files = sorted(config.data_dir.glob("flex_*.xml"))
+        if xml_files:
+            latest = xml_files[-1]
+            latest_file = str(latest)
+            latest_mtime = latest.stat().st_mtime
+    return {
+        "configured": True,
+        "query_id": config.query_id,
+        "data_dir": str(config.data_dir),
+        "latest_file": latest_file,
+        "latest_timestamp": latest_mtime,
+    }
+
+
+def _http_get(url: str, params: dict[str, str]) -> bytes:
+    full_url = f"{url}?{urllib.parse.urlencode(params)}"
+    request = urllib.request.Request(
+        full_url,
+        headers={"User-Agent": "IB-Dashboard/1.0"},
+    )
+    with urllib.request.urlopen(request, timeout=30) as response:
+        return response.read()
+
+
+def _parse_send_request(xml_bytes: bytes) -> tuple[str, str]:
+    root = ET.fromstring(xml_bytes)
+    status = (root.findtext("Status") or "").strip()
+    if status.lower() != "success":
+        raise FlexQueryError(root.findtext("ErrorMessage") or "Flex send-request failed")
+    reference_code = (root.findtext("ReferenceCode") or "").strip()
+    if not reference_code:
+        raise FlexQueryError("Flex send-request returned no ReferenceCode")
+    return status, reference_code
+
+
+def _parse_statement_status(xml_bytes: bytes) -> tuple[bool, str]:
+    root = ET.fromstring(xml_bytes)
+    if root.tag == "FlexStatementResponse":
+        error_message = (root.findtext("ErrorMessage") or "").strip()
+        if error_message:
+            raise FlexQueryError(error_message)
+        return False, ""
+    return True, xml_bytes.decode("utf-8", errors="replace")
+
+
+def fetch_flex_statement() -> dict:
+    config = load_flex_config()
+    if config is None:
+        raise FlexQueryError("Flex Query is not configured")
+
+    config.data_dir.mkdir(parents=True, exist_ok=True)
+
+    _, reference_code = _parse_send_request(_http_get(
+        FLEX_SEND_REQUEST_URL,
+        {"t": config.token, "q": config.query_id, "v": "3"},
+    ))
+
+    statement_xml = None
+    for _ in range(config.max_polls):
+        xml_bytes = _http_get(
+            FLEX_GET_STATEMENT_URL,
+            {"t": config.token, "q": reference_code, "v": "3"},
+        )
+        done, payload = _parse_statement_status(xml_bytes)
+        if done:
+            statement_xml = payload
+            break
+        time.sleep(config.poll_interval_seconds)
+
+    if statement_xml is None:
+        raise FlexQueryError("Timed out waiting for Flex statement generation")
+
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    destination = config.data_dir / f"flex_{timestamp}.xml"
+    destination.write_text(statement_xml, encoding="utf-8")
+
+    return {
+        "saved_to": str(destination),
+        "reference_code": reference_code,
+        "bytes": len(statement_xml.encode("utf-8")),
+    }

backend/ib_client.py

@@ -8,17 +8,17 @@ Notes:
 - Keep behavior simple: subscribe to market data for provided symbols.
 - Store latest quotes in `self.quotes` and positions in `self.positions`.
 """
-import logging
-import threading
-import time
-from typing import Dict, List
+import logging
+import threading
+import time
+from typing import Dict, List
 
 from ibapi.client import EClient
 from ibapi.contract import Contract
 from ibapi.wrapper import EWrapper
 
 
-class IBClient(EWrapper, EClient):
+class IBClient(EWrapper, EClient):
     def __init__(self, host: str = "127.0.0.1", port: int = 7496, client_id: int = 1):
         EClient.__init__(self, self)
         # normalize/validate connection params
@@ -40,18 +40,153 @@ class IBClient(EWrapper, EClient):
         self._connected_event = threading.Event()
 
         # subscriptions requested before connection
-        self._pending_subscriptions = set()
+        self._pending_subscriptions: Dict[str, Dict] = {}
 
         # in-memory stores
         self.quotes: Dict[str, Dict] = {}
         self.positions: List[Dict] = []
+        # store executions (append as execDetails callbacks arrive)
+        self.executions: List[Dict] = []
+        self._execution_ids = set()
+        self.historical_closes: Dict[str, float] = {}
+
+        # tracking for subscriptions: reqId -> symbol
+        self._req_map: Dict[int, str] = {}
+        self._historical_req_map: Dict[int, str] = {}
+        self._requested_symbols = set()
+        self._requested_accounts = set()
+        self._requested_historical = set()
 
-        # tracking for subscriptions: reqId -> symbol
-        self._req_map: Dict[int, str] = {}
-        self._requested_symbols = set()
-
-        # basic logging
-        logging.basicConfig(level=logging.INFO)
+        # basic logging
+        logging.basicConfig(level=logging.INFO)
+
+    def _normalize_expiry(self, expiry):
+        expiry = str(expiry or "").strip()
+        if len(expiry) == 8 and expiry.isdigit():
+            return f"{expiry[0:4]}-{expiry[4:6]}-{expiry[6:8]}"
+        if len(expiry) == 6 and expiry.isdigit():
+            return f"{expiry[0:4]}-{expiry[4:6]}"
+        return expiry
+
+    def _contract_key(self, contract) -> str:
+        if not contract:
+            return ""
+        con_id = getattr(contract, "conId", None)
+        if con_id:
+            return f"CONID:{con_id}"
+        sec_type = (getattr(contract, "secType", None) or "").upper()
+        symbol = getattr(contract, "symbol", None) or ""
+        if sec_type == "OPT":
+            expiry = self._normalize_expiry(getattr(contract, "lastTradeDateOrContractMonth", None))
+            strike = getattr(contract, "strike", None)
+            right = getattr(contract, "right", None) or ""
+            multiplier = getattr(contract, "multiplier", None) or ""
+            return f"OPT:{symbol}:{expiry}:{strike}:{right}:{multiplier}"
+        return f"{sec_type or 'STK'}:{symbol}"
+
+    def _contract_to_record(self, contract) -> Dict:
+        if not contract:
+            return {}
+        sec_type = (getattr(contract, "secType", None) or "STK").upper()
+        symbol = getattr(contract, "symbol", None) or ""
+        if sec_type == "OPT":
+            expiry = self._normalize_expiry(getattr(contract, "lastTradeDateOrContractMonth", None))
+            strike = getattr(contract, "strike", None)
+            right = (getattr(contract, "right", None) or "").upper()
+            multiplier = getattr(contract, "multiplier", None)
+        else:
+            expiry = None
+            strike = None
+            right = None
+            multiplier = None
+        local_symbol = getattr(contract, "localSymbol", None)
+        rec = {
+            "instrument_key": self._contract_key(contract),
+            "con_id": getattr(contract, "conId", None) or None,
+            "symbol": symbol,
+            "security_type": sec_type,
+            "underlying_symbol": symbol,
+            "expiry": expiry or None,
+            "strike": float(strike) if strike not in (None, "") else None,
+            "option_type": right or None,
+            "multiplier": str(multiplier) if multiplier not in (None, "") else None,
+            "local_symbol": local_symbol or None,
+            "exchange": getattr(contract, "exchange", None) or None,
+            "primary_exchange": getattr(contract, "primaryExchange", None) or None,
+            "currency": getattr(contract, "currency", None) or None,
+            "trading_class": getattr(contract, "tradingClass", None) or None,
+        }
+        rec["display_symbol"] = self._format_display_symbol(rec)
+        return rec
+
+    def _format_display_symbol(self, rec: Dict) -> str:
+        sec_type = (rec.get("security_type") or "STK").upper()
+        symbol = rec.get("underlying_symbol") or rec.get("symbol") or ""
+        if sec_type == "OPT":
+            expiry = rec.get("expiry") or "?"
+            strike = rec.get("strike")
+            if isinstance(strike, float) and strike.is_integer():
+                strike_str = str(int(strike))
+            else:
+                strike_str = str(strike) if strike is not None else "?"
+            option_type = rec.get("option_type") or "?"
+            return f"{symbol} {expiry} {strike_str} {option_type}"
+        return symbol
+
+    def _price_multiplier(self, rec: Dict) -> float:
+        sec_type = (rec.get("security_type") or "STK").upper()
+        if sec_type != "OPT":
+            return 1.0
+        multiplier = rec.get("multiplier")
+        try:
+            return float(multiplier) if multiplier not in (None, "", 0, "0") else 100.0
+        except Exception:
+            return 100.0
+
+    def _make_contract_from_record(self, rec: Dict) -> Contract:
+        sec_type = (rec.get("security_type") or "STK").upper()
+        con_id = rec.get("con_id")
+        if con_id:
+            c = Contract()
+            c.conId = int(con_id)
+            c.secType = sec_type
+            c.exchange = rec.get("exchange") or "SMART"
+            currency = rec.get("currency")
+            if currency:
+                c.currency = currency
+            return c
+        if sec_type == "OPT":
+            c = Contract()
+            c.symbol = rec.get("underlying_symbol") or rec.get("symbol")
+            c.secType = "OPT"
+            c.exchange = rec.get("exchange") or "SMART"
+            c.currency = rec.get("currency") or "USD"
+            expiry = rec.get("expiry") or ""
+            c.lastTradeDateOrContractMonth = expiry.replace("-", "")
+            strike = rec.get("strike")
+            if strike is not None:
+                c.strike = float(strike)
+            option_type = rec.get("option_type")
+            if option_type:
+                c.right = option_type
+            multiplier = rec.get("multiplier")
+            if multiplier:
+                c.multiplier = str(multiplier)
+            local_symbol = rec.get("local_symbol")
+            if local_symbol:
+                c.localSymbol = local_symbol
+            primary_exchange = rec.get("primary_exchange")
+            if primary_exchange:
+                c.primaryExchange = primary_exchange
+            trading_class = rec.get("trading_class")
+            if trading_class:
+                c.tradingClass = trading_class
+            return c
+        return self._make_stock_contract(
+            rec.get("symbol") or rec.get("underlying_symbol"),
+            exchange=rec.get("exchange") or "SMART",
+            currency=rec.get("currency") or "USD",
+        )
 
     def connect_and_start(self) -> None:
         if not self.isConnected():
@@ -80,52 +215,64 @@ class IBClient(EWrapper, EClient):
         except Exception:
             pass
         # process any pending subscriptions
-        if self._pending_subscriptions:
-            pending = list(self._pending_subscriptions)
-            self._pending_subscriptions.clear()
-            for s in pending:
-                try:
-                    self.subscribe_market_data(s)
-                except Exception:
-                    logging.exception("Failed to process pending subscription %s", s)
+        if self._pending_subscriptions:
+            pending = list(self._pending_subscriptions.values())
+            self._pending_subscriptions.clear()
+            for s in pending:
+                try:
+                    self.subscribe_market_data(s)
+                except Exception:
+                    logging.exception("Failed to process pending subscription %s", s)
 
     def error(self, reqId, errorCode, errorString):
         logging.error("IB error (req=%s code=%s): %s", reqId, errorCode, errorString)
 
-    def tickPrice(self, reqId, tickType, price, attrib):
-        # store last price when available
-        if price is None:
-            return
-        symbol = self._req_map.get(reqId)
-        if not symbol:
-            return
-        data = self.quotes.setdefault(symbol, {})
-        data["last"] = price
-
-    def tickSize(self, reqId, tickType, size):
-        symbol = self._req_map.get(reqId)
-        if not symbol:
-            return
-        data = self.quotes.setdefault(symbol, {})
-        data["size"] = size
+    def tickPrice(self, reqId, tickType, price, attrib):
+        # store last or closing price when available
+        if price is None or price == -1:
+            return
+        instrument_key = self._req_map.get(reqId)
+        if not instrument_key:
+            return
+        data = self.quotes.setdefault(instrument_key, {})
+        if tickType in (4, 68):
+            data["last"] = price
+        elif tickType in (9, 75):
+            data["close"] = price
+        elif tickType in (1, 66):
+            data["bid"] = price
+        elif tickType in (2, 67):
+            data["ask"] = price
+        elif tickType == 37:
+            data["mark"] = price
+
+    def tickSize(self, reqId, tickType, size):
+        instrument_key = self._req_map.get(reqId)
+        if not instrument_key:
+            return
+        data = self.quotes.setdefault(instrument_key, {})
+        data["size"] = size
 
     def updatePortfolio(self, contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName):
         # simple representation of the position
-        if not contract or not contract.symbol:
-            return
-        pos = {
-            "symbol": contract.symbol,
-            "position": position,
-            "marketPrice": marketPrice,
-            "marketValue": marketValue,
-        }
-        # replace existing entry for the symbol
-        for i, p in enumerate(self.positions):
-            if p.get("symbol") == pos["symbol"]:
-                self.positions[i] = pos
-                break
-        else:
-            self.positions.append(pos)
+        if not contract or not contract.symbol:
+            return
+        contract_rec = self._contract_to_record(contract)
+        pos = {
+            **contract_rec,
+            "position": position,
+            "marketPrice": marketPrice,
+            "marketValue": marketValue,
+            "averageCost": averageCost,
+            "account": accountName,
+        }
+        # replace existing entry for the symbol
+        for i, p in enumerate(self.positions):
+            if p.get("instrument_key") == pos["instrument_key"] and p.get("account") == accountName:
+                self.positions[i] = pos
+                break
+        else:
+            self.positions.append(pos)
 
     def positionEnd(self):
         # called when a batch of position updates is finished
@@ -136,25 +283,58 @@ class IBClient(EWrapper, EClient):
 
         Signature from EWrapper: position(self, account, contract, position, avgCost)
         """
-        try:
-            if not contract or not getattr(contract, 'symbol', None):
-                return
-            pos = {
-                "symbol": contract.symbol,
-                "position": position,
-                "averageCost": avgCost,
-                "account": account,
-            }
-            # replace existing entry for the symbol
-            for i, p in enumerate(self.positions):
-                if p.get("symbol") == pos["symbol"] and p.get("account") == account:
-                    self.positions[i] = pos
-                    break
-            else:
-                self.positions.append(pos)
+        try:
+            if not contract or not getattr(contract, 'symbol', None):
+                return
+            contract_rec = self._contract_to_record(contract)
+            pos = {
+                **contract_rec,
+                "position": position,
+                "averageCost": avgCost,
+                "account": account,
+            }
+            # replace existing entry for the symbol
+            for i, p in enumerate(self.positions):
+                if p.get("instrument_key") == pos["instrument_key"] and p.get("account") == account:
+                    self.positions[i] = pos
+                    break
+            else:
+                self.positions.append(pos)
         except Exception:
             logging.exception("Error handling position callback")
 
+    def execDetails(self, reqId, contract, execution):
+        """Capture execution details for building lots.
+
+        `execution` is an `Execution` object from ibapi which contains
+        .shares, .price, .side, .execId, .time (may be empty)
+        """
+        try:
+            symbol = getattr(contract, "symbol", None)
+            if not symbol:
+                return
+            contract_rec = self._contract_to_record(contract)
+            entry = {
+                "execId": getattr(execution, "execId", None),
+                **contract_rec,
+                "shares": float(getattr(execution, "shares", 0)),
+                "price": float(getattr(execution, "price", 0.0)),
+                "side": getattr(execution, "side", ""),
+                "time": getattr(execution, "time", None),
+                "permId": getattr(execution, "permId", None),
+            }
+            exec_id = entry.get("execId")
+            if exec_id and exec_id in self._execution_ids:
+                return
+            if exec_id:
+                self._execution_ids.add(exec_id)
+            self.executions.append(entry)
+        except Exception:
+            logging.exception("Failed to record execution detail")
+
+    def execDetailsEnd(self, reqId):
+        logging.info("Finished receiving executions (req=%s). %d execs total", reqId, len(self.executions))
+
     # ---- helpers ----
     def _new_req_id(self) -> int:
         rid = self._next_req_id
@@ -169,38 +349,45 @@ class IBClient(EWrapper, EClient):
         c.currency = currency
         return c
 
-    def subscribe_market_data(self, symbol: str) -> None:
-        """Subscribe to basic market data for `symbol` (in USD on SMART exchange).
+    def subscribe_market_data(self, contract_or_record) -> None:
+        """Subscribe to basic market data for a contract or simple symbol string.
+
+        Repeated calls for the same symbol are ignored.
+        """
+        if isinstance(contract_or_record, str):
+            contract = self._make_stock_contract(contract_or_record)
+            queued_record = {"symbol": contract_or_record, "security_type": "STK"}
+            instrument_key = self._contract_key(contract)
+        else:
+            contract = self._make_contract_from_record(contract_or_record)
+            queued_record = dict(contract_or_record)
+            instrument_key = self._contract_key(contract)
+        if instrument_key in self._requested_symbols:
+            return
+        # if not connected yet, queue the subscription
+        if not self._connected_event.is_set():
+            logging.info("Not connected yet; queuing subscription for %s", instrument_key)
+            self._pending_subscriptions[instrument_key] = queued_record
+            # ensure we start connection
+            try:
+                self.connect_and_start()
+            except Exception:
+                logging.exception("Failed to start connection while queuing subscription")
+            return
+
+        req_id = self._new_req_id()
+        self._req_map[req_id] = instrument_key
+        # prefer delayed data when live market data is not available
+        try:
+            self.reqMarketDataType(3)
+            self.reqMktData(req_id, contract, "", False, False, [])
+            self._requested_symbols.add(instrument_key)
+            logging.info("Subscribed to market data for %s (req=%s)", instrument_key, req_id)
+        except Exception as e:
+            logging.exception("Failed to subscribe to %s: %s", instrument_key, e)
 
-        Repeated calls for the same symbol are ignored.
-        """
-        if symbol in self._requested_symbols:
-            return
-        # if not connected yet, queue the subscription
-        if not self._connected_event.is_set():
-            logging.info("Not connected yet; queuing subscription for %s", symbol)
-            self._pending_subscriptions.add(symbol)
-            # ensure we start connection
-            try:
-                self.connect_and_start()
-            except Exception:
-                logging.exception("Failed to start connection while queuing subscription")
-            return
-
-        req_id = self._new_req_id()
-        self._req_map[req_id] = symbol
-        contract = self._make_stock_contract(symbol)
-        # request real-time market data (empty generic tick list)
-        try:
-            self.reqMarketDataType(1)
-            self.reqMktData(req_id, contract, "", False, False, [])
-            self._requested_symbols.add(symbol)
-            logging.info("Subscribed to market data for %s (req=%s)", symbol, req_id)
-        except Exception as e:
-            logging.exception("Failed to subscribe to %s: %s", symbol, e)
-
-    def request_positions(self) -> None:
-        """Ask the IB API to send current positions (triggers updatePortfolio callbacks)."""
+    def request_positions(self) -> None:
+        """Ask the IB API to send current positions (triggers updatePortfolio callbacks)."""
         if not self._connected_event.is_set():
             logging.info("Not connected yet; will request positions only if connection parameters are valid")
             # validate host/port before trying to connect
@@ -221,13 +408,316 @@ class IBClient(EWrapper, EClient):
             threading.Thread(target=_delayed, daemon=True).start()
             return
 
-        try:
-            self.reqPositions()
-        except Exception:
-            logging.exception("Failed to request positions")
+        try:
+            self.reqPositions()
+        except Exception:
+            logging.exception("Failed to request positions")
+
+    def request_account_updates(self, account: str) -> None:
+        """Subscribe to portfolio/account updates for a specific account."""
+        if not account or account in self._requested_accounts:
+            return
+        if not self._connected_event.is_set():
+            try:
+                self.connect_and_start()
+            except Exception:
+                logging.exception("Failed to start connection for account updates")
+            return
+        try:
+            self.reqAccountUpdates(True, account)
+            self._requested_accounts.add(account)
+        except Exception:
+            logging.exception("Failed to request account updates for %s", account)
 
-    def get_quotes(self) -> Dict[str, Dict]:
-        return self.quotes
+    def request_executions(self, start_time: str | None = None) -> None:
+        """Request executions from IB (will trigger execDetails callbacks).
+
+        Uses an empty ExecutionFilter to fetch recent executions. The results
+        arrive via `execDetails` callbacks.
+        """
+        try:
+            # import here to avoid module-level dependency issues
+            from ibapi.execution import ExecutionFilter
+            filt = ExecutionFilter()
+            if start_time:
+                filt.time = start_time
+            rid = self._new_req_id()
+            self.reqExecutions(rid, filt)
+        except Exception:
+            logging.exception("Failed to request executions")
+
+    def historicalData(self, reqId, bar):
+        try:
+            instrument_key = self._historical_req_map.get(reqId)
+            if not instrument_key:
+                return
+            close = getattr(bar, "close", None)
+            if close not in (None, -1):
+                self.historical_closes[instrument_key] = float(close)
+        except Exception:
+            logging.exception("Failed to process historical data")
+
+    def historicalDataEnd(self, reqId, start, end):
+        self._historical_req_map.pop(reqId, None)
+
+    def request_historical_close(self, contract_or_record) -> None:
+        if isinstance(contract_or_record, str):
+            contract = self._make_stock_contract(contract_or_record)
+        else:
+            contract = self._make_contract_from_record(contract_or_record)
+        instrument_key = self._contract_key(contract)
+        if not instrument_key or instrument_key in self._requested_historical:
+            return
+        if not self._connected_event.is_set():
+            try:
+                self.connect_and_start()
+            except Exception:
+                logging.exception("Failed to start connection for historical close")
+            return
+        try:
+            req_id = self._new_req_id()
+            self._historical_req_map[req_id] = instrument_key
+            self._requested_historical.add(instrument_key)
+            self.reqHistoricalData(req_id, contract, "", "2 D", "1 day", "TRADES", 0, 1, False, [])
+        except Exception:
+            logging.exception("Failed to request historical close for %s", instrument_key)
+
+    def get_quotes(self):
+        return dict(self.quotes)
+
+    def get_positions(self):
+        return list(self.positions)
+
+    def compute_open_lots(self, include_position_fallback: bool = False):
+        """Compute open lots using FIFO matching from recorded executions.
+
+        Returns a list of lots with: symbol, qty, cost_basis, avg_price, market_price,
+        market_value, gain_dollars, gain_percent, cagr_percent, buy_time
+        """
+        # group executions by symbol
+        from collections import defaultdict
+        from datetime import datetime
 
-    def get_positions(self) -> List[Dict]:
-        return self.positions
+        by_symbol = defaultdict(list)
+        for e in self.executions:
+            by_symbol[e["instrument_key"]].append(e)
+
+        # include symbols from positions as well so we can fallback
+        position_symbols = set()
+        position_by_symbol = {}
+        for p in self.positions:
+            instrument_key = p.get("instrument_key")
+            if not instrument_key:
+                continue
+            position_symbols.add(instrument_key)
+            if instrument_key not in position_by_symbol:
+                position_by_symbol[instrument_key] = p
+
+        lots = []
+        all_symbols = set(by_symbol.keys()) | position_symbols
+        for instrument_key in all_symbols:
+            execs = by_symbol.get(instrument_key, [])
+            position = position_by_symbol.get(instrument_key) or {}
+            meta = execs[0] if execs else position
+            # sort by time if available, else keep order
+            def _time_key(x):
+                t = x.get("time")
+                if not t:
+                    return 0
+                # try parsing common formats
+                for fmt in ("%Y%m%d  %H:%M:%S", "%Y%m%d %H:%M:%S", "%Y%m%d-%H:%M:%S"):
+                    try:
+                        return datetime.strptime(t, fmt).timestamp()
+                    except Exception:
+                        continue
+                try:
+                    return float(t)
+                except Exception:
+                    return 0
+
+            execs_sorted = sorted(execs, key=_time_key) if execs else []
+            # build open lots using FIFO for both long and short positions
+            open_longs = []
+            open_shorts = []
+            for e in execs_sorted:
+                side = (e.get("side") or "").upper()
+                shares = abs(float(e.get("shares", 0) or 0))
+                price = float(e.get("price", 0.0) or 0.0)
+                time_str = e.get("time")
+                if side == "BOT" or side == "BUY":
+                    qty_to_buy = shares
+                    while qty_to_buy > 0 and open_shorts:
+                        s = open_shorts[0]
+                        matched = min(qty_to_buy, abs(s["qty"]))
+                        s["qty"] += matched
+                        qty_to_buy -= matched
+                        if abs(s["qty"]) < 1e-9:
+                            open_shorts.pop(0)
+                    if qty_to_buy > 0:
+                        open_longs.append({"qty": qty_to_buy, "price": price, "time": time_str})
+                else:
+                    qty_to_sell = shares
+                    while qty_to_sell > 0 and open_longs:
+                        b = open_longs[0]
+                        matched = min(qty_to_sell, b["qty"])
+                        b["qty"] -= matched
+                        qty_to_sell -= matched
+                        if b["qty"] < 1e-9:
+                            open_longs.pop(0)
+                    if qty_to_sell > 0:
+                        open_shorts.append({"qty": -qty_to_sell, "price": price, "time": time_str})
+
+            # remaining buys are open lots
+            # ensure market data available for symbol
+            if instrument_key not in self.quotes:
+                try:
+                    self.subscribe_market_data(meta)
+                except Exception:
+                    pass
+            market_price = None
+            q = self.quotes.get(instrument_key)
+            if q:
+                market_price = q.get("last")
+                if market_price is None:
+                    market_price = q.get("close")
+                if market_price is None:
+                    market_price = q.get("mark")
+            if market_price is None:
+                market_price = self.historical_closes.get(instrument_key)
+            if market_price is None:
+                market_price = position.get("marketPrice")
+
+            try:
+                position_qty = float(position.get("position") or 0.0)
+            except Exception:
+                position_qty = 0.0
+            try:
+                position_market_value = float(position.get("marketValue")) if position.get("marketValue") is not None else None
+            except Exception:
+                position_market_value = None
+
+            known_market_value_remaining = position_market_value
+            first_exec_time = execs_sorted[0].get("time") if execs_sorted else None
+            price_multiplier = self._price_multiplier(meta or position)
+
+            if execs_sorted:
+                open_lots = open_longs + open_shorts
+                for b in open_lots:
+                    qty = float(b["qty"])
+                    avg_price = float(b["price"]) * price_multiplier
+                    cost_basis = qty * avg_price
+                    effective_market_price = None if market_price is None else market_price * price_multiplier
+                    market_value = None if effective_market_price is None else qty * effective_market_price
+                    if market_value is None and known_market_value_remaining is not None and position_qty:
+                        market_value = known_market_value_remaining * (qty / position_qty)
+                    if known_market_value_remaining is not None and market_value is not None:
+                        known_market_value_remaining -= market_value
+                    gain_dollars = None if market_value is None else market_value - cost_basis
+                    gain_percent = None
+                    cagr_percent = None
+                    base_cost = abs(cost_basis)
+                    if base_cost and market_value is not None:
+                        try:
+                            gain_percent = (gain_dollars / base_cost) * 100
+                        except Exception:
+                            gain_percent = None
+                    # estimate CAGR using buy time if available
+                    buy_time = b.get("time")
+                    if buy_time and market_value is not None and base_cost > 0:
+                        try:
+                            # parse time similar to above
+                            for fmt in ("%Y%m%d  %H:%M:%S", "%Y%m%d %H:%M:%S", "%Y%m%d-%H:%M:%S"):
+                                try:
+                                    dt = datetime.strptime(buy_time, fmt)
+                                    break
+                                except Exception:
+                                    dt = None
+                            if dt is None:
+                                # try numeric timestamp
+                                try:
+                                    dt = datetime.fromtimestamp(float(buy_time))
+                                except Exception:
+                                    dt = None
+                            if dt is not None:
+                                years = max((datetime.utcnow() - dt).days / 365.0, 1e-6)
+                                try:
+                                    ending_value = base_cost + gain_dollars
+                                    if ending_value > 0:
+                                        cagr = (ending_value / base_cost) ** (1.0 / years) - 1.0
+                                        cagr_percent = cagr * 100.0
+                                    else:
+                                        cagr_percent = None
+                                except Exception:
+                                    cagr_percent = None
+                        except Exception:
+                            cagr_percent = None
+
+                    lots.append({
+                        "symbol": meta.get("symbol"),
+                        "display_symbol": meta.get("display_symbol") or self._format_display_symbol(meta),
+                        "security_type": meta.get("security_type"),
+                        "underlying_symbol": meta.get("underlying_symbol"),
+                        "expiry": meta.get("expiry"),
+                        "strike": meta.get("strike"),
+                        "option_type": meta.get("option_type"),
+                        "instrument_key": instrument_key,
+                        "qty": qty,
+                        "avg_price": avg_price,
+                        "cost_basis": cost_basis,
+                        "market_price": effective_market_price,
+                        "market_value": market_value,
+                        "gain_dollars": gain_dollars,
+                        "gain_percent": gain_percent,
+                        "cagr_percent": cagr_percent,
+                        "buy_time": buy_time,
+                    })
+            elif include_position_fallback:
+                # fallback: use positions data to create a single lot per symbol
+                for p in self.positions:
+                    if p.get("instrument_key") != instrument_key:
+                        continue
+                    qty = p.get("position") or 0
+                    avg_price = p.get("averageCost") or p.get("average_cost") or 0.0
+                    try:
+                        qtyf = float(qty)
+                    except Exception:
+                        qtyf = 0.0
+                    try:
+                        avg_pricef = float(avg_price)
+                    except Exception:
+                        avg_pricef = 0.0
+                    cost_basis = qtyf * avg_pricef
+                    effective_market_price = None if market_price is None else market_price * self._price_multiplier(p)
+                    if effective_market_price is not None:
+                        market_value = qtyf * effective_market_price
+                    else:
+                        market_value = position_market_value
+                    gain_dollars = None if market_value is None else market_value - cost_basis
+                    gain_percent = None
+                    base_cost = abs(cost_basis)
+                    if base_cost and market_value is not None:
+                        try:
+                            gain_percent = (gain_dollars / base_cost) * 100
+                        except Exception:
+                            gain_percent = None
+                    lots.append({
+                        "symbol": p.get("symbol"),
+                        "display_symbol": p.get("display_symbol") or self._format_display_symbol(p),
+                        "security_type": p.get("security_type"),
+                        "underlying_symbol": p.get("underlying_symbol"),
+                        "expiry": p.get("expiry"),
+                        "strike": p.get("strike"),
+                        "option_type": p.get("option_type"),
+                        "instrument_key": instrument_key,
+                        "qty": qtyf,
+                        "avg_price": avg_pricef,
+                        "cost_basis": cost_basis,
+                        "market_price": effective_market_price,
+                        "market_value": market_value,
+                        "gain_dollars": gain_dollars,
+                        "gain_percent": gain_percent,
+                        "cagr_percent": None,
+                        "buy_time": first_exec_time,
+                    })
+
+        return lots