Update app.py

app.py

@@ -1,13 +1,14 @@
 import os
 import sys
+import traceback
 from pathlib import Path
-from typing import Tuple, Any, List
+from typing import List, Tuple, Any
 
 import duckdb
 import pandas as pd
 import numpy as np
 import matplotlib
-matplotlib.use("Agg")  # headless backend for Spaces
+matplotlib.use("Agg")  # headless for Spaces
 import matplotlib.pyplot as plt
 import gradio as gr
 
@@ -15,7 +16,7 @@ import gradio as gr
 # Basic configuration
 # =========================
 APP_TITLE = "ALCO Liquidity & Interest-Rate Risk Dashboard"
-TABLE_FQN = "my_db.main.masterdataset_v"   # your source table
+TABLE_FQN = "my_db.main.masterdataset_v"   # source table
 VIEW_FQN = "my_db.main.positions_v"        # normalized view created by this app
 EXPORT_DIR = Path("exports")
 EXPORT_DIR.mkdir(exist_ok=True)
@@ -32,35 +33,18 @@ PRODUCT_SOF = [
 # =========================
 # Helpers
 # =========================
-def safe_float(x, default: float = 0.0) -> float:
-    try:
-        if x is None or (isinstance(x, float) and np.isnan(x)):
-            return default
-        return float(x)
-    except Exception:
-        return default
-
-def zeros_like_index(index) -> pd.Series:
-    return pd.Series([0] * len(index), index=index)
-
 def connect_md() -> duckdb.DuckDBPyConnection:
     token = os.environ.get("MOTHERDUCK_TOKEN", "")
     if not token:
-        raise RuntimeError("MOTHERDUCK_TOKEN is not set. Add it in your Space → Settings → Secrets.")
-    try:
-        return duckdb.connect(f"md:?motherduck_token={token}")
-    except Exception as e:
-        raise RuntimeError(f"MotherDuck connection failed: {e}") from e
+        raise RuntimeError("MOTHERDUCK_TOKEN is not set. Add it in Space → Settings → Secrets.")
+    return duckdb.connect(f"md:?motherduck_token={token}")
 
 def discover_columns(conn: duckdb.DuckDBPyConnection, table_fqn: str) -> List[str]:
-    q = f"""
-    SELECT lower(column_name) AS col
-    FROM information_schema.columns
-    WHERE table_schema = split_part('{table_fqn}', '.', 2)
-      AND table_name   = split_part('{table_fqn}', '.', 3)
-    """
-    df = conn.execute(q).fetchdf()
-    return df["col"].tolist()
+    # More robust than information_schema across DuckDB/MotherDuck
+    df = conn.execute(f"DESCRIBE {table_fqn};").fetchdf()
+    # DuckDB: columns listed under 'column_name'
+    name_col = "column_name" if "column_name" in df.columns else df.columns[0]
+    return [str(c).lower() for c in df[name_col].tolist()]
 
 def build_view_sql(existing_cols: List[str]) -> str:
     wanted = [
@@ -68,28 +52,24 @@ def build_view_sql(existing_cols: List[str]) -> str:
         "currency", "Portfolio_value", "Interest_rate",
         "days_to_maturity"
     ]
-    select_list = []
+    sel = []
     for c in wanted:
         if c.lower() in existing_cols:
-            select_list.append(c)
+            sel.append(c)
         else:
-            # fill missing columns with typed NULLs
             if c in ("Portfolio_value", "Interest_rate", "days_to_maturity", "months"):
-                select_list.append(f"CAST(NULL AS DOUBLE) AS {c}")
+                sel.append(f"CAST(NULL AS DOUBLE) AS {c}")
             else:
-                select_list.append(f"CAST(NULL AS VARCHAR) AS {c}")
+                sel.append(f"CAST(NULL AS VARCHAR) AS {c}")
 
     sof_list = ", ".join([f"'{p}'" for p in PRODUCT_SOF])
     asset_list = ", ".join([f"'{p}'" for p in PRODUCT_ASSETS])
-
     bucket_case = (
-        f"CASE "
-        f"WHEN lower(product) IN ({sof_list}) THEN 'SoF' "
+        f"CASE WHEN lower(product) IN ({sof_list}) THEN 'SoF' "
         f"WHEN lower(product) IN ({asset_list}) THEN 'Assets' "
         f"ELSE 'Unknown' END AS bucket"
     )
-
-    select_sql = ",\n  ".join(select_list + [bucket_case])
+    select_sql = ",\n  ".join(sel + [bucket_case])
     return f"""
     CREATE OR REPLACE VIEW {VIEW_FQN} AS
     SELECT
@@ -97,87 +77,36 @@ def build_view_sql(existing_cols: List[str]) -> str:
     FROM {TABLE_FQN};
     """
 
-def ensure_view(conn: duckdb.DuckDBPyConnection, existing_cols: List[str]):
+def ensure_view(conn: duckdb.DuckDBPyConnection, cols: List[str]) -> None:
     required = {"product", "portfolio_value", "days_to_maturity"}
-    if not required.issubset(set(existing_cols)):
+    if not required.issubset(set(cols)):
         raise RuntimeError(
-            f"Source table {TABLE_FQN} must contain columns {sorted(required)}; "
-            f"found {sorted(existing_cols)}"
+            f"Source table {TABLE_FQN} must contain columns {sorted(required)}; found {sorted(cols)}"
         )
-    conn.execute(build_view_sql(existing_cols))
-
-def fetch_kpis(conn: duckdb.DuckDBPyConnection) -> Tuple[float, float, float]:
-    sql = f"""
-    SELECT
-      COALESCE(SUM(CASE WHEN bucket='Assets' AND days_to_maturity<=1 THEN Portfolio_value END),0) AS assets_t1,
-      COALESCE(SUM(CASE WHEN bucket='SoF'    AND days_to_maturity<=1 THEN Portfolio_value END),0) AS sof_t1,
-      COALESCE(SUM(CASE WHEN bucket='Assets' AND days_to_maturity<=1 THEN Portfolio_value END),0)
-      - COALESCE(SUM(CASE WHEN bucket='SoF'  AND days_to_maturity<=1 THEN Portfolio_value END),0) AS net_gap_t1
-    FROM {VIEW_FQN};
-    """
-    df = conn.execute(sql).fetchdf()
-    if df.empty:
-        return 0.0, 0.0, 0.0
-    return safe_float(df["assets_t1"].iloc[0]), safe_float(df["sof_t1"].iloc[0]), safe_float(df["net_gap_t1"].iloc[0])
+    conn.execute(build_view_sql(cols))
 
-def fetch_ladder(conn: duckdb.DuckDBPyConnection) -> pd.DataFrame:
-    sql = f"""
-    SELECT
-      CASE
-        WHEN days_to_maturity <= 1 THEN 'T+1'
-        WHEN days_to_maturity BETWEEN 2 AND 7 THEN 'T+2..7'
-        WHEN days_to_maturity BETWEEN 8 AND 30 THEN 'T+8..30'
-        ELSE 'T+31+'
-      END AS time_bucket,
-      bucket,
-      SUM(Portfolio_value) AS amount
-    FROM {VIEW_FQN}
-    GROUP BY 1,2
-    ORDER BY 1,2;
-    """
-    df = conn.execute(sql).fetchdf()
-    if df.empty:
-        return pd.DataFrame({"time_bucket": [], "bucket": [], "amount": []})
-    return df
+def safe_num(x) -> float:
+    try:
+        return float(0.0 if x is None or (isinstance(x, float) and np.isnan(x)) else x)
+    except Exception:
+        return 0.0
 
-def fetch_irr(conn: duckdb.DuckDBPyConnection, cols: List[str]) -> pd.DataFrame:
-    has_months = "months" in cols
-    has_ir = "interest_rate" in cols
-    t_expr = "CASE WHEN days_to_maturity IS NOT NULL THEN days_to_maturity/365.0"
-    if has_months:
-        t_expr += " WHEN months IS NOT NULL THEN months/12.0"
-    t_expr += " ELSE NULL END"
-    y_expr = "(Interest_rate/100.0)" if has_ir else "0.0"
-    sql = f"""
-    SELECT
-      bucket,
-      SUM(Portfolio_value) AS pv_sum,
-      SUM(Portfolio_value * {t_expr}) / NULLIF(SUM(Portfolio_value),0) AS dur_mac,
-      SUM(Portfolio_value * ({t_expr})/(1+({y_expr}))) / NULLIF(SUM(Portfolio_value),0) AS dur_mod
-    FROM {VIEW_FQN}
-    GROUP BY bucket;
-    """
-    df = conn.execute(sql).fetchdf()
-    if df.empty:
-        return pd.DataFrame({"bucket": [], "pv_sum": [], "dur_mac": [], "dur_mod": []})
-    return df
+def zeros_like_index(index) -> pd.Series:
+    return pd.Series([0] * len(index), index=index)
 
 def plot_ladder(df: pd.DataFrame):
     try:
-        if df.empty:
+        if df is None or df.empty:
             fig, ax = plt.subplots(figsize=(7, 3))
-            ax.text(0.5, 0.5, "No data", ha="center", va="center", fontsize=12)
+            ax.text(0.5, 0.5, "No data", ha="center", va="center")
             ax.axis("off")
             return fig
-
         pivot = df.pivot(index="time_bucket", columns="bucket", values="amount").fillna(0)
         order = ["T+1", "T+2..7", "T+8..30", "T+31+"]
         pivot = pivot.reindex(order)
         fig, ax = plt.subplots(figsize=(7, 4))
-
         assets = pivot["Assets"] if "Assets" in pivot.columns else zeros_like_index(pivot.index)
         sof = pivot["SoF"] if "SoF" in pivot.columns else zeros_like_index(pivot.index)
-
         ax.bar(pivot.index, assets, label="Assets")
         ax.bar(pivot.index, -sof, label="SoF")
         ax.axhline(0, color="gray", lw=1)
@@ -187,89 +116,132 @@ def plot_ladder(df: pd.DataFrame):
         fig.tight_layout()
         return fig
     except Exception as e:
-        # Return a simple figure with the error rendered
         fig, ax = plt.subplots(figsize=(7, 3))
         ax.text(0.01, 0.8, "Chart Error:", fontsize=12, ha="left")
         ax.text(0.01, 0.5, str(e), fontsize=10, ha="left", wrap=True)
         ax.axis("off")
         return fig
 
-def export_excel(as_of_date: str,
-                 assets_t1: float,
-                 sof_t1: float,
-                 net_gap_t1: float,
-                 ladder: pd.DataFrame,
-                 irr: pd.DataFrame) -> Path:
-    out = EXPORT_DIR / f"alco_report_{as_of_date}.xlsx"
+# =========================
+# Query fragments
+# =========================
+KPI_SQL = f"""
+SELECT
+  COALESCE(SUM(CASE WHEN bucket='Assets' AND days_to_maturity<=1 THEN Portfolio_value END),0) AS assets_t1,
+  COALESCE(SUM(CASE WHEN bucket='SoF'    AND days_to_maturity<=1 THEN Portfolio_value END),0) AS sof_t1,
+  COALESCE(SUM(CASE WHEN bucket='Assets' AND days_to_maturity<=1 THEN Portfolio_value END),0)
+  - COALESCE(SUM(CASE WHEN bucket='SoF'  AND days_to_maturity<=1 THEN Portfolio_value END),0) AS net_gap_t1
+FROM {VIEW_FQN};
+"""
+
+LADDER_SQL = f"""
+SELECT
+  CASE
+    WHEN days_to_maturity <= 1 THEN 'T+1'
+    WHEN days_to_maturity BETWEEN 2 AND 7 THEN 'T+2..7'
+    WHEN days_to_maturity BETWEEN 8 AND 30 THEN 'T+8..30'
+    ELSE 'T+31+'
+  END AS time_bucket,
+  bucket,
+  SUM(Portfolio_value) AS amount
+FROM {VIEW_FQN}
+GROUP BY 1,2
+ORDER BY 1,2;
+"""
+
+def irr_sql(cols: List[str]) -> str:
+    has_months = "months" in cols
+    has_ir = "interest_rate" in cols
+    t_expr = "CASE WHEN days_to_maturity IS NOT NULL THEN days_to_maturity/365.0"
+    if has_months:
+        t_expr += " WHEN months IS NOT NULL THEN months/12.0"
+    t_expr += " ELSE NULL END"
+    y_expr = "(Interest_rate/100.0)" if has_ir else "0.0"
+    return f"""
+    SELECT
+      bucket,
+      SUM(Portfolio_value) AS pv_sum,
+      SUM(Portfolio_value * {t_expr}) / NULLIF(SUM(Portfolio_value),0) AS dur_mac,
+      SUM(Portfolio_value * ({t_expr})/(1+({y_expr}))) / NULLIF(SUM(Portfolio_value),0) AS dur_mod
+    FROM {VIEW_FQN}
+    GROUP BY bucket;
+    """
+
+def export_excel(as_of: str, assets_t1: float, sof_t1: float, net_gap_t1: float,
+                 ladder: pd.DataFrame, irr: pd.DataFrame) -> Path:
+    out = EXPORT_DIR / f"alco_report_{as_of or 'NA'}.xlsx"
     with pd.ExcelWriter(out, engine="xlsxwriter") as xw:
         pd.DataFrame({
-            "as_of_date": [as_of_date],
+            "as_of_date": [as_of or "N/A"],
             "assets_t1": [assets_t1],
             "sof_t1": [sof_t1],
             "net_gap_t1": [net_gap_t1],
         }).to_excel(xw, index=False, sheet_name="kpis")
-        ladder.to_excel(xw, index=False, sheet_name="ladder")
-        irr.to_excel(xw, index=False, sheet_name="irr")
+        (ladder if ladder is not None else pd.DataFrame()).to_excel(xw, index=False, sheet_name="ladder")
+        (irr if irr is not None else pd.DataFrame()).to_excel(xw, index=False, sheet_name="irr")
     return out
 
 # =========================
-# Gradio UI logic
+# Dashboard callback
 # =========================
-def run_dashboard():
+def run_dashboard() -> Tuple[str, str, str, str, str, Any, pd.DataFrame, pd.DataFrame, str]:
     """
     Returns:
-      status (str),
-      as_of (str),
-      assets_t1 (float),
-      sof_t1 (float),
-      net_gap_t1 (float),
-      fig (matplotlib fig),
-      ladder_df (DataFrame),
-      irr_df (DataFrame),
-      excel_file (path str)
+      status, as_of, assets_t1, sof_t1, net_gap_t1, figure, ladder_df, irr_df, excel_path
+      (all text values are returned as strings to avoid component type errors)
     """
-    status = "✅ OK"
     try:
         conn = connect_md()
-        cols = discover_columns(conn, TABLE_FQN)  # lower-cased names
+
+        # Discover columns & build view
+        cols = discover_columns(conn, TABLE_FQN)
         ensure_view(conn, cols)
 
-        # As-of when available (otherwise N/A)
+        # as_of (optional)
         as_of = "N/A"
         if "as_of_date" in cols:
             tmp = conn.execute(f"SELECT max(as_of_date) AS d FROM {VIEW_FQN}").fetchdf()
             if not tmp.empty and not pd.isna(tmp["d"].iloc[0]):
                 as_of = str(tmp["d"].iloc[0])[:10]
 
-        assets_t1, sof_t1, net_gap_t1 = fetch_kpis(conn)
-        ladder = fetch_ladder(conn)
-        irr = fetch_irr(conn, cols)
+        # KPIs
+        kpi = conn.execute(KPI_SQL).fetchdf()
+        assets_t1 = safe_num(kpi["assets_t1"].iloc[0]) if not kpi.empty else 0.0
+        sof_t1    = safe_num(kpi["sof_t1"].iloc[0]) if not kpi.empty else 0.0
+        net_gap   = safe_num(kpi["net_gap_t1"].iloc[0]) if not kpi.empty else 0.0
+
+        # Ladder & IRR
+        ladder = conn.execute(LADDER_SQL).fetchdf()
+        irr    = conn.execute(irr_sql(cols)).fetchdf()
 
         fig = plot_ladder(ladder)
-        xlsx_path = export_excel(as_of, assets_t1, sof_t1, net_gap_t1, ladder, irr)
+        xlsx = export_excel(as_of, assets_t1, sof_t1, net_gap, ladder, irr)
+        xlsx_str = str(xlsx if xlsx.exists() else "")
 
+        status = "✅ OK"
         return (
             status,
             as_of,
-            assets_t1,
-            sof_t1,
-            net_gap_t1,
+            f"{assets_t1:,.0f}",
+            f"{sof_t1:,.0f}",
+            f"{net_gap:,.0f}",
             fig,
             ladder,
             irr,
-            str(xlsx_path),
+            xlsx_str,
         )
+
     except Exception as e:
-        # Swallow the error for the UI; show user-friendly message + zeros/empty placeholders
-        status = f"❌ Error: {e}"
+        tb = traceback.format_exc()
+        # Return placeholders + human-readable error in status
         empty_df = pd.DataFrame()
         fig = plot_ladder(empty_df)
         return (
-            status,
+            f"❌ Error: {e}\n\n{tb}",
             "N/A",
-            0.0,
-            0.0,
-            0.0,
+            "0",
+            "0",
+            "0",
             fig,
             empty_df,
             empty_df,
@@ -282,17 +254,18 @@ def run_dashboard():
 with gr.Blocks(title=APP_TITLE) as demo:
     gr.Markdown(f"# {APP_TITLE}\n_Source:_ `{TABLE_FQN}` → `{VIEW_FQN}`")
 
-    status = gr.Textbox(label="Status", interactive=False)
+    status = gr.Textbox(label="Status", interactive=False, lines=5)
 
     with gr.Row():
         refresh_btn = gr.Button("🔄 Refresh", variant="primary")
 
     with gr.Row():
         as_of = gr.Textbox(label="As of date", interactive=False)
+
     with gr.Row():
-        a1 = gr.Number(label="Assets T+1 (LKR)", precision=0)
-        a2 = gr.Number(label="SoF T+1 (LKR)", precision=0)
-        a3 = gr.Number(label="Net Gap T+1 (LKR)", precision=0)
+        a1 = gr.Textbox(label="Assets T+1 (LKR)", interactive=False)
+        a2 = gr.Textbox(label="SoF T+1 (LKR)", interactive=False)
+        a3 = gr.Textbox(label="Net Gap T+1 (LKR)", interactive=False)
 
     chart = gr.Plot(label="Maturity Ladder")
     ladder_df = gr.Dataframe(label="Ladder Detail")