Add incremental delta refresh and fix Decimal/datetime cache binding

src/sqlmem/cache.py

@@ -8,8 +8,9 @@ from pathlib import Path
 from loguru import logger
 
 import sqlmem._meta as _meta
+from ._coerce import coerce_params, coerce_row
 
-SCHEMA_VERSION = 2
+SCHEMA_VERSION = 3
 
 
 class CacheManager:
@@ -41,7 +42,8 @@ class CacheManager:
                 table_name      TEXT PRIMARY KEY,
                 last_refresh_at TEXT NOT NULL,
                 row_count       INTEGER,
-                is_full         INTEGER NOT NULL DEFAULT 0
+                is_full         INTEGER NOT NULL DEFAULT 0,
+                last_synced_at  TEXT
             );
             CREATE TABLE IF NOT EXISTS _sqlmem_columns (
                 table_name  TEXT NOT NULL,
@@ -159,18 +161,103 @@ class CacheManager:
         cols = ", ".join(columns)
         logger.info(f"Fetching {table!r} columns [{cols}] from source DB")
         rows = source_conn.execute(f"SELECT {cols} FROM {table}").fetchall()
+        clean_rows = [coerce_row(row) for row in rows]
 
         with self._lock:
             self._mem_conn.execute(f"DROP TABLE IF EXISTS {table}")
             col_defs = ", ".join(f"{c} TEXT" for c in columns)
             self._mem_conn.execute(f"CREATE TABLE {table} ({col_defs})")
             placeholders = ", ".join("?" * len(columns))
-            self._mem_conn.executemany(f"INSERT INTO {table} VALUES ({placeholders})", rows)
+            self._mem_conn.executemany(f"INSERT INTO {table} VALUES ({placeholders})", clean_rows)
             self._mem_conn.commit()
 
         self.mark_table_refreshed(table, len(rows), full)
         logger.info(f"Table {table!r} cached ({len(rows)} rows, columns: {columns})")
 
+    def execute_in_memory(
+        self, sql: str, params: tuple | list | dict | None = None
+    ) -> tuple[list[str], list[tuple]]:
+        """Run a read query against the in-memory cache, serialized with writers."""
+        bound = coerce_params(params)
+        with self._lock:
+            cursor = self._mem_conn.execute(sql) if bound is None else self._mem_conn.execute(sql, bound)
+            col_names = [desc[0] for desc in cursor.description]
+            rows = cursor.fetchall()
+        return col_names, rows
+
+    # --- delta refresh support ---------------------------------------------
+
+    def get_table_columns(self, table: str) -> list[str]:
+        """Authoritative ordered column list of a cached table (via PRAGMA)."""
+        rows = self._mem_conn.execute(f"PRAGMA table_info({table})").fetchall()
+        return [r[1] for r in rows]
+
+    def create_unique_index(self, table: str, key_columns: list[str]) -> None:
+        """Create the unique index on *key_columns* that makes upsert-by-key work."""
+        cols = ", ".join(key_columns)
+        index = f"idx_{table}_pk"
+        with self._lock:
+            self._mem_conn.execute(
+                f"CREATE UNIQUE INDEX IF NOT EXISTS {index} ON {table} ({cols})"
+            )
+            self._mem_conn.commit()
+
+    def get_last_synced_at(self, table: str) -> str | None:
+        row = self._mem_conn.execute(
+            "SELECT last_synced_at FROM _sqlmem_tables WHERE table_name = ?", (table,)
+        ).fetchone()
+        return row[0] if row else None
+
+    def set_last_synced_at(self, table: str, value: str | None) -> None:
+        with self._lock:
+            self._mem_conn.execute(
+                "UPDATE _sqlmem_tables SET last_synced_at = ? WHERE table_name = ?",
+                (value, table),
+            )
+            self._mem_conn.commit()
+
+    def max_value(self, table: str, column: str) -> str | None:
+        """Maximum value of *column* across cached rows (the delta watermark)."""
+        row = self._mem_conn.execute(f"SELECT MAX({column}) FROM {table}").fetchone()
+        return row[0] if row else None
+
+    def upsert_rows(self, table: str, columns: list[str], rows: list[tuple]) -> None:
+        """Insert-or-replace *rows* by the table's unique key, then refresh row_count."""
+        col_list = ", ".join(columns)
+        placeholders = ", ".join("?" * len(columns))
+        clean_rows = [coerce_row(row) for row in rows]
+        with self._lock:
+            self._mem_conn.executemany(
+                f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({placeholders})",
+                clean_rows,
+            )
+            self._mem_conn.commit()
+            count = self._mem_conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
+        self.mark_table_refreshed(table, count, self.is_table_full(table))
+
+    def reset(self) -> None:
+        """Wipe the entire cache — every cached table plus the on-disk file."""
+        logger.info("Resetting cache — dropping all cached tables.")
+        with self._lock:
+            user_tables = [
+                r[0]
+                for r in self._mem_conn.execute(
+                    "SELECT name FROM sqlite_master "
+                    r"WHERE type = 'table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\' "
+                r"AND name NOT LIKE '\_sqlmem\_%' ESCAPE '\'"
+                ).fetchall()
+            ]
+            for name in user_tables:
+                self._mem_conn.execute(f"DROP TABLE IF EXISTS {name}")
+            self._mem_conn.execute("DELETE FROM _sqlmem_tables")
+            self._mem_conn.execute("DELETE FROM _sqlmem_columns")
+            self._mem_conn.commit()
+        try:
+            if self._db_path.exists():
+                self._db_path.unlink()
+        except OSError as e:
+            logger.error(f"Failed to delete cache file {self._db_path}: {e}")
+
     def close(self) -> None:
         self._backup_to_disk()
         self._closed = True