Batch large-table loads to bound memory and add per-table state to stats

src/sqlmem/stats.py

@@ -3,11 +3,23 @@ import threading
 from dataclasses import dataclass
 
 
+class TableState:
+    """Live processing state of a cached table (value of ``TableStats.state``)."""
+
+    LOADING = "loading"        # a full load is in progress
+    REFRESHING = "refreshing"  # an incremental (delta) refresh is in progress
+    READY = "ready"            # cached and idle
+    STALE = "stale"            # TTL expired — will reload on next access
+    ERROR = "error"            # the last load failed
+
+
 @dataclass(frozen=True)
 class TableStats:
     rows: int
     columns: list[str]
     last_refresh: str
+    state: str = TableState.READY
+    tracking: str = "static"  # "delta" | "ttl" | "static"
 
 
 @dataclass(frozen=True)
@@ -37,14 +49,19 @@ class StatsCollector:
         with self._lock:
             self.refetches += 1
 
-    def snapshot(self, conn: sqlite3.Connection) -> Stats:
+    def snapshot(
+        self, conn: sqlite3.Connection, states: dict[str, str] | None = None
+    ) -> Stats:
+        states = states or {}
         with self._lock:
             hits, misses, refetches = self.hits, self.misses, self.refetches
 
         tables: dict[str, TableStats] = {}
+        cached: set[str] = set()
         for table_name, row_count, last_refresh in conn.execute(
             "SELECT table_name, row_count, last_refresh_at FROM _sqlmem_tables"
         ).fetchall():
+            cached.add(table_name)
             columns = [
                 r[0]
                 for r in conn.execute(
@@ -56,6 +73,12 @@ class StatsCollector:
                 rows=row_count or 0,
                 columns=columns,
                 last_refresh=last_refresh,
+                state=states.get(table_name, TableState.READY),
             )
 
+        # Surface tables that are mid-first-load (not yet in _sqlmem_tables) or failed.
+        for name, state in states.items():
+            if name not in cached and state in (TableState.LOADING, TableState.ERROR):
+                tables[name] = TableStats(rows=0, columns=[], last_refresh="", state=state)
+
         return Stats(hits=hits, misses=misses, refetches=refetches, tables=tables)