Add scheduled refresh for tables that change at a fixed time of day

src/sqlmem/__init__.py

@@ -8,7 +8,7 @@ from .config import DEBUG
 from .delta import DeltaConfig
 from .engine import CachingEngine
 from .exceptions import ReadOnlyError, UndeclaredError, UnsupportedQueryError
-from .spec import TTL, Delta, TableSpec
+from .spec import TTL, Delta, Schedule, TableSpec
 from .stats import Stats, TableStats
 
 _DEFAULT_FORMAT = (
@@ -62,6 +62,7 @@ __all__ = [
     "DeltaConfig",
     "Delta",
     "TTL",
+    "Schedule",
     "TableSpec",
     "ReadOnlyError",
     "UnsupportedQueryError",

src/sqlmem/cache.py

@@ -325,15 +325,21 @@ class CacheManager:
             ).fetchone()
         return bool(row and row[0])
 
-    def seconds_since_refresh(self, table: str) -> float | None:
-        """Age of a cached table in seconds, or None if it is not cached."""
+    def last_refresh_at(self, table: str) -> datetime | None:
+        """When a cached table was last (re)loaded, or None if it is not cached."""
         with self._lock:
             row = self._conn.execute(
                 "SELECT last_refresh_at FROM _sqlmem_tables WHERE table_name = ?", (table,)
             ).fetchone()
         if not row or not row[0]:
             return None
-        last = datetime.fromisoformat(row[0])
+        return datetime.fromisoformat(row[0])
+
+    def seconds_since_refresh(self, table: str) -> float | None:
+        """Age of a cached table in seconds, or None if it is not cached."""
+        last = self.last_refresh_at(table)
+        if last is None:
+            return None
         return (datetime.now(timezone.utc) - last).total_seconds()
 
     def discover_columns(self, table: str, source_conn: sqlite3.Connection) -> list[str]:

src/sqlmem/engine.py

@@ -1,5 +1,6 @@
 import threading
 from dataclasses import replace
+from itertools import combinations
 from pathlib import Path
 from typing import Any
 
@@ -22,7 +23,8 @@ from .exceptions import UndeclaredError
 from .executor import QueryExecutor
 from .parser import Params, ParsedQuery, parse
 from .registry import ColumnRegistry
-from .spec import TTL, TableSpec
+from .schedule import ScheduleSpec, is_due, now_local, parse_schedule, seconds_until_next
+from .spec import TTL, Schedule, TableSpec
 from .stats import Stats, StatsCollector, TableState, TableStats
 
 
@@ -31,18 +33,21 @@ def _specs_to_config(
 ) -> tuple[
     dict[str, DeltaConfig],
     dict[str, int],
+    dict[str, ScheduleSpec],
     dict[str, list[str | list[str]]],
     dict[str, list[str]],
     dict[str, list[str] | None],
 ]:
     """Convert declarative ``TableSpec``s into the engine's internal config dicts.
 
-    Returns ``(delta, ttl, indexes, datetime_columns, declared)`` — the first four
-    mirror the legacy kwargs; ``declared`` maps each table to its allowed columns
-    (``None`` = whole table / any column) for fail-fast query checking.
+    Returns ``(delta, ttl, schedule, indexes, datetime_columns, declared)`` — the
+    first five mirror the legacy kwargs; ``declared`` maps each table to its
+    allowed columns (``None`` = whole table / any column) for fail-fast query
+    checking.
     """
     delta: dict[str, DeltaConfig] = {}
     ttl: dict[str, int] = {}
+    schedule: dict[str, ScheduleSpec] = {}
     indexes: dict[str, list[str | list[str]]] = {}
     datetime_columns: dict[str, list[str]] = {}
     declared: dict[str, list[str] | None] = {}
@@ -57,9 +62,11 @@ def _specs_to_config(
         refresh = spec.refresh
         if isinstance(refresh, TTL):
             ttl[spec.name] = refresh.seconds
+        elif isinstance(refresh, Schedule):
+            schedule[spec.name] = refresh.times
         elif isinstance(refresh, DeltaConfig):
             delta[spec.name] = refresh
-    return delta, ttl, indexes, datetime_columns, declared
+    return delta, ttl, schedule, indexes, datetime_columns, declared
 
 
 class _LazySource:
@@ -96,6 +103,7 @@ class CachingEngine:
         source_engine: Engine,
         delta: dict[str, DeltaConfig] | None = None,
         ttl: dict[str, int] | None = None,
+        schedule: dict[str, ScheduleSpec] | None = None,
         indexes: dict[str, list[str | list[str]]] | None = None,
         in_memory: bool | None = None,
         cache_db_path: str | Path | None = None,
@@ -112,17 +120,19 @@ class CachingEngine:
         self._source_engine = source_engine
 
         # Declarative mode: a list of TableSpecs is converted to the same internal
-        # config the legacy delta=/ttl=/indexes=/datetime_columns= kwargs produce,
-        # plus a declared-columns allowlist (for fail-fast) and preload set.
+        # config the legacy delta=/ttl=/schedule=/indexes=/datetime_columns= kwargs
+        # produce, plus a declared-columns allowlist (for fail-fast) and preload set.
         self._declared: dict[str, list[str] | None] | None = None
         self._preload_specs: list[TableSpec] = []
         if tables is not None:
-            if any(x is not None for x in (delta, ttl, indexes, datetime_columns)):
+            if any(x is not None for x in (delta, ttl, schedule, indexes, datetime_columns)):
                 raise ValueError(
                     "Pass either tables=[TableSpec(...)] or the legacy "
-                    "delta=/ttl=/indexes=/datetime_columns= kwargs, not both."
+                    "delta=/ttl=/schedule=/indexes=/datetime_columns= kwargs, not both."
                 )
-            delta, ttl, indexes, datetime_columns, self._declared = _specs_to_config(tables)
+            delta, ttl, schedule, indexes, datetime_columns, self._declared = _specs_to_config(
+                tables
+            )
             self._preload_specs = [s for s in tables if s.preload]
 
         use_memory = IN_MEMORY if in_memory is None else in_memory
@@ -144,18 +154,13 @@ class CachingEngine:
         self._stats = StatsCollector()
         self._delta = self._resolve_delta(delta or {})
         self._ttl = dict(ttl or {})
+        self._schedule = {t: parse_schedule(t, s) for t, s in (schedule or {}).items()}
         self._index_columns = self._register_indexes(indexes or {})
         self._refresher = DeltaRefresher(self._cache, self._delta)
 
-        overlap = set(self._delta) & set(self._ttl)
-        if overlap:
-            raise ValueError(
-                f"Tables {sorted(overlap)} are in both delta and ttl — a table is "
-                "either delta-refreshed (has a change column) or TTL-refreshed (full "
-                "reload), not both."
-            )
+        self._check_one_method_per_table()
 
-        if self._delta or self._ttl or self._preload_specs:
+        if self._delta or self._ttl or self._schedule or self._preload_specs:
             # Startup work (preload of declared tables + delta/TTL catch-up for
             # tables restored from disk) can take a while on a cold start. By
             # default it runs on the background thread so it never blocks
@@ -168,6 +173,24 @@ class CachingEngine:
 
         logger.info("CachingEngine initialized.")
 
+    def _check_one_method_per_table(self) -> None:
+        """Reject a table configured with more than one refresh method.
+
+        Per-table this is already impossible via ``TableSpec.refresh`` (one
+        field); this guards the legacy kwargs, where ``delta=``/``ttl=``/
+        ``schedule=`` are independent dicts a caller could overlap by mistake.
+        """
+        methods = {"delta": set(self._delta), "ttl": set(self._ttl), "schedule": set(self._schedule)}
+        for (name_a, tables_a), (name_b, tables_b) in combinations(methods.items(), 2):
+            overlap = tables_a & tables_b
+            if overlap:
+                raise ValueError(
+                    f"Tables {sorted(overlap)} are in both {name_a} and {name_b} — a table "
+                    "has exactly one refresh method: delta (incremental, via a change "
+                    "column), ttl (full reload once older than a max age) or schedule "
+                    "(full reload at fixed times of day)."
+                )
+
     def _register_indexes(
         self, indexes: dict[str, list[str | list[str]]]
     ) -> dict[str, list[str]]:
@@ -227,19 +250,19 @@ class CachingEngine:
         errors: dict[str, TableError],
         last_runs: dict[str, str],
     ) -> TableStats:
-        """Annotate a TableStats with refresh tracking, TTL staleness, errors and run time."""
+        """Annotate a TableStats with refresh tracking, staleness, errors and run time."""
         if name in self._delta:
             tracking = "delta"
         elif name in self._ttl:
             tracking = "ttl"
+        elif name in self._schedule:
+            tracking = "schedule"
         else:
             tracking = "static"
 
         state = table_stats.state
-        if state == TableState.READY and name in self._ttl:
-            age = self._cache.seconds_since_refresh(name)
-            if age is not None and age > self._ttl[name]:
-                state = TableState.STALE
+        if state == TableState.READY and self._is_stale(name):
+            state = TableState.STALE
 
         last_refresh = last_runs.get(name)
         err = errors.get(name)
@@ -255,6 +278,15 @@ class CachingEngine:
             )
         return replace(table_stats, tracking=tracking, state=state, last_refresh=last_refresh)
 
+    def _is_stale(self, name: str) -> bool:
+        """True if a cached table is past its TTL or its last scheduled reload time."""
+        if name in self._ttl:
+            age = self._cache.seconds_since_refresh(name)
+            return age is not None and age > self._ttl[name]
+        if name in self._schedule:
+            return is_due(self._schedule[name], self._cache.last_refresh_at(name), now_local())
+        return False
+
     def _make_executor(self, source: Any) -> QueryExecutor:
         return QueryExecutor(
             self._cache,
@@ -264,6 +296,7 @@ class CachingEngine:
             self._delta,
             self._ttl,
             self._index_columns,
+            self._schedule,
         )
 
     def _check_declared(self, parsed: ParsedQuery) -> None:
@@ -329,6 +362,7 @@ class CachingEngine:
                 raw_conn = sa_conn.connection.dbapi_connection
                 self._refresher.refresh(raw_conn)
                 self._refresh_ttl(raw_conn)
+                self._refresh_scheduled(raw_conn)
         except Exception as e:
             logger.error(f"Refresh cycle failed: {e}")
 
@@ -341,26 +375,57 @@ class CachingEngine:
             if age is None or age <= ttl:
                 continue
             try:
-                columns = self._cache.get_table_columns(table)
-                full = self._cache.is_table_full(table)
-                self._cache.load_table(table, columns, source_conn, full=full)
+                self._reload(table, source_conn)
                 logger.info(f"TTL refresh {table!r}: reloaded (age {age:.0f}s > {ttl}s)")
             except Exception as e:
                 logger.error(f"TTL refresh failed for {table!r}: {e}")
 
+    def _refresh_scheduled(self, source_conn: Any) -> None:
+        """Full-reload scheduled tables whose scheduled time of day has passed."""
+        now = now_local()
+        for table, times in self._schedule.items():
+            if not self._cache.is_table_cached(table):
+                continue
+            if not is_due(times, self._cache.last_refresh_at(table), now):
+                continue
+            try:
+                self._reload(table, source_conn)
+                logger.info(f"Scheduled refresh {table!r}: reloaded (slot passed)")
+            except Exception as e:
+                logger.error(f"Scheduled refresh failed for {table!r}: {e}")
+
+    def _reload(self, table: str, source_conn: Any) -> None:
+        """Full-reload a cached table, keeping its cached column set and full status."""
+        columns = self._cache.get_table_columns(table)
+        full = self._cache.is_table_full(table)
+        self._cache.load_table(table, columns, source_conn, full=full)
+
     def _start_refresh_thread(self, initial_catch_up: bool = True) -> None:
         def loop() -> None:
             if initial_catch_up:
                 self._preload()      # off-main-thread declared-table preload
                 self._run_refresh()  # off-main-thread startup catch-up
             event = threading.Event()
-            while not event.wait(self._refresh_interval):
+            while not event.wait(self._next_tick()):
                 self._run_refresh()
 
         t = threading.Thread(target=loop, daemon=True, name="sqlmem-delta")
         t.start()
         logger.debug(f"Delta refresh thread started (interval={self._refresh_interval}s)")
 
+    def _next_tick(self) -> float:
+        """Seconds to sleep before the next refresh cycle.
+
+        Normally the fixed refresh interval, but shortened when a scheduled
+        table's time of day falls sooner, so a schedule fires within a second
+        of its slot instead of waiting for the next tick.
+        """
+        if not self._schedule:
+            return float(self._refresh_interval)
+        # +1s so the wake-up lands just past the slot, never a hair before it.
+        soonest = min(seconds_until_next(t, now_local()) for t in self._schedule.values()) + 1.0
+        return max(1.0, min(float(self._refresh_interval), soonest))
+
     def invalidate(self, table: str) -> None:
         logger.info(f"Manually invalidating cache for table {table!r}")
         with self._cache._lock:

src/sqlmem/executor.py

@@ -1,4 +1,5 @@
 from collections.abc import Callable
+from datetime import time
 from typing import Any
 
 from loguru import logger
@@ -7,6 +8,7 @@ from .cache import CacheManager
 from .delta import ResolvedDelta
 from .parser import ParsedQuery
 from .registry import ColumnRegistry
+from .schedule import is_due, now_local
 from .stats import StatsCollector
 
 
@@ -20,6 +22,7 @@ class QueryExecutor:
         delta: dict[str, ResolvedDelta] | None = None,
         ttl: dict[str, int] | None = None,
         index_columns: dict[str, list[str]] | None = None,
+        schedule: dict[str, tuple[time, ...]] | None = None,
     ) -> None:
         self._cache = cache
         self._registry = registry
@@ -28,14 +31,25 @@ class QueryExecutor:
         self._delta = delta or {}
         self._ttl = ttl or {}
         self._index_columns = index_columns or {}
+        self._schedule = schedule or {}
 
-    def _ttl_expired(self, table: str) -> bool:
-        """True if *table* has a TTL and its cached copy is older than that TTL."""
+    def _stale_reason(self, table: str) -> str | None:
+        """Why *table*'s cached copy must be reloaded before answering a query.
+
+        Returns ``"ttl"``/``"schedule"`` or ``None`` (fresh) — the single check
+        both ``_ensure_full``/``_ensure_columns`` and the double-checked-locking
+        ``satisfied`` predicates use, so TTL and schedule expiry are treated
+        identically everywhere staleness matters.
+        """
         ttl = self._ttl.get(table)
-        if ttl is None:
-            return False
-        age = self._cache.seconds_since_refresh(table)
-        return age is not None and age > ttl
+        if ttl is not None:
+            age = self._cache.seconds_since_refresh(table)
+            if age is not None and age > ttl:
+                return "ttl"
+        times = self._schedule.get(table)
+        if times is not None and is_due(times, self._cache.last_refresh_at(table), now_local()):
+            return "schedule"
+        return None
 
     def execute(self, parsed: ParsedQuery) -> list[dict]:
         for table in parsed.tables:
@@ -63,31 +77,31 @@ class QueryExecutor:
             self._ensure_columns(table, parsed.columns_by_table[table])
 
     def _full_satisfied(self, table: str) -> bool:
-        """True if *table* is cached in full and not TTL-expired (a SELECT * hit)."""
+        """True if *table* is cached in full and not expired (a SELECT * hit)."""
         return (
             self._cache.is_table_cached(table)
             and self._cache.is_table_full(table)
-            and not self._ttl_expired(table)
+            and self._stale_reason(table) is None
         )
 
     def _columns_satisfied(self, table: str, columns: list[str]) -> bool:
-        """True if *table* is cached with all *columns* present and not TTL-expired."""
-        if not self._cache.is_table_cached(table) or self._ttl_expired(table):
+        """True if *table* is cached with all *columns* present and not expired."""
+        if not self._cache.is_table_cached(table) or self._stale_reason(table) is not None:
             return False
         return set(columns).issubset(self._cache.get_table_columns(table))
 
     def _ensure_full(self, table: str) -> None:
         """Load every column of *table* (SELECT * / t.*), refetching unless already full."""
         cached = self._cache.is_table_cached(table)
-        stale = cached and self._ttl_expired(table)
+        reason = self._stale_reason(table) if cached else None
 
-        if cached and self._cache.is_table_full(table) and not stale:
+        if cached and self._cache.is_table_full(table) and not reason:
             logger.debug(f"Cache hit (full): {table!r}")
             self._stats.record_hit()
             return
 
-        if cached and stale:
-            logger.info(f"Cache expired (ttl) — reloading {table!r} in full.")
+        if reason:
+            logger.info(f"Cache expired ({reason}) — reloading {table!r} in full.")
             self._stats.record_refetch()
         elif cached:
             logger.warning(f"Re-fetching {table!r} in full — SELECT * requested.")
@@ -99,18 +113,18 @@ class QueryExecutor:
         self._load(table, columns, full=True, satisfied=lambda cols: self._full_satisfied(table))
 
     def _ensure_columns(self, table: str, columns: list[str]) -> None:
-        """Load *table* with at least *columns*, refetching on new columns or TTL expiry."""
+        """Load *table* with at least *columns*, refetching on new columns or expiry."""
         missing = self._registry.needs_refetch(table, columns)
         table_cached = self._cache.is_table_cached(table)
-        stale = table_cached and self._ttl_expired(table)
+        reason = self._stale_reason(table) if table_cached else None
 
-        if table_cached and not missing and not stale:
+        if table_cached and not missing and not reason:
             logger.debug(f"Cache hit: {table!r} columns={columns}")
             self._stats.record_hit()
             return
 
-        if stale:
-            logger.info(f"Cache expired (ttl) — reloading {table!r}.")
+        if reason:
+            logger.info(f"Cache expired ({reason}) — reloading {table!r}.")
             self._stats.record_refetch()
         elif table_cached and missing:
             logger.warning(
@@ -122,7 +136,7 @@ class QueryExecutor:
             self._stats.record_miss()
 
         all_columns = list(self._registry.get_columns(table)) + missing
-        # Preserve a fully-cached table's status across a TTL reload.
+        # Preserve a fully-cached table's status across a TTL/schedule reload.
         full = table_cached and self._cache.is_table_full(table)
         self._load(
             table,

src/sqlmem/schedule.py

@@ -0,0 +1,87 @@
+from datetime import datetime, time, timedelta, timezone
+
+ScheduleSpec = str | time | list[str | time] | tuple[str | time, ...]
+"""One or more times of day, as ``"HH:MM"`` / ``"HH:MM:SS"`` strings or :class:`~datetime.time`."""
+
+
+def parse_schedule(table: str, spec: ScheduleSpec) -> tuple[time, ...]:
+    """Normalize a per-table schedule into a sorted tuple of times of day.
+
+    Accepts a single time or a list of them; raises ``ValueError`` on an empty
+    spec or an unparsable / timezone-aware time.
+    """
+    if isinstance(spec, (str, time)):
+        items: list[str | time] = [spec]
+    elif isinstance(spec, (list, tuple)):
+        items = list(spec)
+    else:
+        raise ValueError(
+            f"Invalid schedule {spec!r} for {table!r} — expected a time of day "
+            "('HH:MM', 'HH:MM:SS' or datetime.time), or a list of them."
+        )
+    if not items:
+        raise ValueError(
+            f"Schedule for {table!r} is empty — give at least one time of day, e.g. '03:00'."
+        )
+    return tuple(sorted({_parse_time(table, item) for item in items}))
+
+
+def _parse_time(table: str, value: str | time) -> time:
+    if isinstance(value, time):
+        parsed = value
+    else:
+        try:
+            parsed = time.fromisoformat(value)
+        except (TypeError, ValueError):
+            raise ValueError(
+                f"Invalid schedule time {value!r} for {table!r} — "
+                "expected 'HH:MM' or 'HH:MM:SS'."
+            ) from None
+    if parsed.tzinfo is not None:
+        raise ValueError(
+            f"Schedule time {value!r} for {table!r} carries a timezone — "
+            "schedule times are interpreted in the local timezone, give them bare."
+        )
+    return parsed.replace(microsecond=0)
+
+
+def now_local() -> datetime:
+    """Current time as a timezone-aware datetime in the local timezone."""
+    return datetime.now(timezone.utc).astimezone()
+
+
+def previous_occurrence(times: tuple[time, ...], now: datetime) -> datetime:
+    """The most recent scheduled moment at or before *now*."""
+    return max(c for c in _candidates(times, now, days=(0, -1)) if c <= now)
+
+
+def next_occurrence(times: tuple[time, ...], now: datetime) -> datetime:
+    """The first scheduled moment strictly after *now*."""
+    return min(c for c in _candidates(times, now, days=(0, 1)) if c > now)
+
+
+def _candidates(
+    times: tuple[time, ...], now: datetime, days: tuple[int, ...]
+) -> list[datetime]:
+    return [
+        datetime.combine(now.date() + timedelta(days=d), t, tzinfo=now.tzinfo)
+        for d in days
+        for t in times
+    ]
+
+
+def seconds_until_next(times: tuple[time, ...], now: datetime) -> float:
+    return (next_occurrence(times, now) - now).total_seconds()
+
+
+def is_due(times: tuple[time, ...], last_refresh: datetime | None, now: datetime) -> bool:
+    """True if the table has not been reloaded since its last scheduled moment.
+
+    Comparing against the *last scheduled occurrence* (rather than tracking fired
+    timers) makes the schedule survive restarts and coarse refresh ticks: a table
+    whose 03:00 slot was missed because the process was down reloads on the first
+    check after start, and never fires twice for the same slot.
+    """
+    if last_refresh is None:  # not cached — the first query loads it anyway
+        return False
+    return last_refresh < previous_occurrence(times, now)

src/sqlmem/spec.py

@@ -10,6 +10,7 @@ kwargs keep working; ``tables=`` is converted to the same internal config.
 from dataclasses import dataclass, field
 
 from .delta import DeltaConfig
+from .schedule import ScheduleSpec, parse_schedule
 
 # Friendly alias for the declarative API; ``Delta`` and ``DeltaConfig`` are the
 # same type (``change_column`` + ``key_columns``), so either may be used as a
@@ -24,6 +25,22 @@ class TTL:
     seconds: int
 
 
+@dataclass(frozen=True)
+class Schedule:
+    """Time-of-day refresh strategy: full-reload the table at fixed times of day.
+
+    *times* is one time of day or a list of them (``"HH:MM"``/``"HH:MM:SS"``
+    strings or :class:`datetime.time`), interpreted in the local timezone.
+    Validated eagerly so a typo surfaces at construction, not at the first
+    refresh tick.
+    """
+
+    times: ScheduleSpec
+
+    def __post_init__(self) -> None:
+        parse_schedule("<Schedule>", self.times)  # eager validation; table name unknown yet
+
+
 @dataclass(frozen=True)
 class TableSpec:
     """Declarative specification of one cached table.
@@ -33,8 +50,9 @@ class TableSpec:
     a query asking for a column outside the list raises
     :class:`~sqlmem.exceptions.UndeclaredError`.
 
-    *refresh* is a :class:`Delta` (change-column incremental sync) or :class:`TTL`
-    (time-based full reload), or ``None`` for a static table loaded once.
+    *refresh* is a :class:`Delta` (change-column incremental sync), :class:`TTL`
+    (time-based full reload) or :class:`Schedule` (full reload at fixed times of
+    day), or ``None`` for a static table loaded once.
 
     *preload=True* loads the table at startup (in the background by default) so the
     first query is a cache hit instead of paying a cold load; a copy already fresh
@@ -44,6 +62,6 @@ class TableSpec:
     name: str
     columns: list[str] | None = None
     indexes: list[str | list[str]] = field(default_factory=list)
-    refresh: DeltaConfig | TTL | None = None
+    refresh: DeltaConfig | TTL | Schedule | None = None
     datetime_columns: list[str] = field(default_factory=list)
     preload: bool = False

src/sqlmem/stats.py

@@ -9,7 +9,7 @@ class TableState:
     LOADING = "loading"        # a full load is in progress
     REFRESHING = "refreshing"  # an incremental (delta) refresh is in progress
     READY = "ready"            # cached and idle
-    STALE = "stale"            # TTL expired — will reload on next access
+    STALE = "stale"            # TTL expired / schedule slot passed — reloads on next access
     ERROR = "error"            # the last load failed
 
 
@@ -25,7 +25,7 @@ class TableStats:
     # the first cycle runs after start. Answers "is the refresh loop alive?".
     last_refresh: str | None = None
     state: str = TableState.READY
-    tracking: str = "static"  # "delta" | "ttl" | "static"
+    tracking: str = "static"  # "delta" | "ttl" | "schedule" | "static"
     # Most recent load/refresh failure for this table, if any. ``consecutive_failures``
     # resets to 0 on the next success, so > 0 means the table is currently failing.
     last_error: str | None = None