fix(recovery): seed a damaged rowid edge from min()/max() before the INT64 domain
When the leftmost (or rightmost) leaf of a table b-tree is damaged, the edge probe `SELECT rowid ... ORDER BY rowid ASC LIMIT 1` walks the table tree and raises, and _salvage_rowid_bounds fell back to INT64_MIN. The gallop from the surviving edge cannot cap that side either (every probe crosses the damaged leaf), so bisection burned the entire 10,000-query budget moving the bound inward by a few thousand rowids out of 9.2e18 and the table was lost — a 4-row gateway_routing table in #98050, sessions + session_model_usage in #100313. `SELECT min(rowid), max(rowid)` is answered by the planner from any covering index (every Hermes table has at least the PRIMARY KEY autoindex) without touching the damaged leaf, which is exactly what the reporter verified by hand. Ask it for the missing edge(s) first; only when it fails too does the domain fallback + gallop run as before. Reported under `aggregate_edges` so recovery.json still shows how the bound was obtained. Live repro (real fixture: leftmost `sessions` leaf cell count overwritten, 400 rows): BEFORE bounds low=-9223372036854775808 copied=0 range_queries=10000 query_limit_reached=True status=failed; AFTER low=1 high=400 copied=391 range_queries=40 status=partial (only the damaged leaf's rows are lost). Refs #98050 Refs #100313 Reported-by: Ace-Kelly Corroborated-by: Proff506
This commit is contained in:
@@ -414,6 +414,22 @@ def _salvage_rowid_bounds(source: sqlite3.Connection, table: str) -> dict[str, A
|
||||
result["empty" if not result["errors"] else "unavailable"] = True
|
||||
return result
|
||||
|
||||
# An ordered LIMIT 1 walks the table b-tree and dies on a damaged edge leaf, while the
|
||||
# aggregate lets the planner answer from any covering index (every Hermes table has at
|
||||
# least a PRIMARY KEY autoindex). Ask it before falling back to the synthetic domain:
|
||||
# bisecting from INT64_MIN burned the whole query budget on a 4-row table (#98050).
|
||||
missing = [edge for edge in ("low", "high") if rows[edge] is None]
|
||||
if missing:
|
||||
try:
|
||||
aggregate = source.execute(f'SELECT min(rowid), max(rowid) FROM "{table}"').fetchone()
|
||||
except sqlite3.DatabaseError as exc:
|
||||
result["errors"].append(f"aggregate rowid bounds: {exc}")
|
||||
else:
|
||||
for edge, value in zip(("low", "high"), aggregate):
|
||||
if rows[edge] is None and value is not None:
|
||||
rows[edge] = int(value)
|
||||
result.setdefault("aggregate_edges", []).append(edge)
|
||||
|
||||
# A damaged edge can stop one ordered probe. Keep the readable edge and bound the other side by the
|
||||
# SQLite rowid domain, so bisection never assumes user databases hold only positive ids.
|
||||
if rows["low"] is None:
|
||||
|
||||
@@ -876,3 +876,42 @@ def test_partial_recovery_skips_phantom_row_rejected_by_destination_schema(
|
||||
with sqlite3.connect(str(output)) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM sessions WHERE id = 'phantom'").fetchone()[0] == 0
|
||||
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 21
|
||||
|
||||
|
||||
|
||||
def test_salvage_bounds_damaged_low_edge_from_the_aggregate_not_the_int64_domain(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""#98050: with the leftmost leaf damaged, ``ORDER BY rowid ASC LIMIT 1`` fails while
|
||||
``min(rowid)`` still answers via the covering index. Bisecting from INT64_MIN burned the
|
||||
whole 10,000-query budget and lost every row; the aggregate must seed the bound instead."""
|
||||
source = tmp_path / "low-edge.db"
|
||||
sessions_root = _make_many_sessions_source(source, session_count=180)
|
||||
page_size, leaf_pages = _btree_leaf_pages(source, sessions_root)
|
||||
assert len(leaf_pages) >= 3
|
||||
first_leaf = leaf_pages[0]
|
||||
data = bytearray(source.read_bytes())
|
||||
header_offset = (first_leaf - 1) * page_size
|
||||
assert data[header_offset] == 0x0D
|
||||
data[header_offset + 3 : header_offset + 5] = b"\xff\xff"
|
||||
source.write_bytes(data)
|
||||
|
||||
conn = sqlite3.connect(str(source))
|
||||
try:
|
||||
with pytest.raises(sqlite3.DatabaseError):
|
||||
conn.execute('SELECT rowid FROM "sessions" ORDER BY rowid ASC LIMIT 1').fetchone()
|
||||
bounds = session_recovery._salvage_rowid_bounds(conn, "sessions")
|
||||
assert bounds["low"] == 1 and bounds["high"] == 180
|
||||
assert bounds["fallback_edges"] == []
|
||||
|
||||
destination = sqlite3.connect(":memory:")
|
||||
destination.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY, source TEXT, started_at REAL)")
|
||||
result = session_recovery._copy_table_salvage(
|
||||
conn, destination, "sessions", chunk_size=16, progress_cb=None, source_rows=180,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
assert result["query_limit_reached"] is False
|
||||
assert result["range_queries"] < 200
|
||||
# Only the rows on the damaged leaf are lost; everything behind it is recovered.
|
||||
assert result["copied_rows"] >= 180 - 60
|
||||
|
||||
@@ -157,9 +157,10 @@ def test_exact_lookup_recovers_tail_row_next_to_damaged_high_edge(
|
||||
|
||||
copied = report["copy"]["messages"]
|
||||
bounds = copied["rowid_bounds"]
|
||||
# Premise check: the high edge probe really failed and fell back.
|
||||
# Premise check: the high edge probe really failed; the bound came from the aggregate
|
||||
# (#98050) or, when that fails too, the synthetic-domain fallback.
|
||||
assert any("high rowid" in error for error in bounds["errors"]), bounds
|
||||
assert "high" in bounds["fallback_edges"]
|
||||
assert "high" in bounds["fallback_edges"] or "high" in bounds.get("aggregate_edges", ())
|
||||
|
||||
conn = sqlite3.connect(str(output))
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user