fix(rerank): derive the doc cap from tokens (measured floor 1.47), cap the query (#9)

docs_mcp/server.py

@@ -57,6 +57,32 @@ RERANK_URL = os.environ.get("RERANK_URL", "").rstrip("/") or None
 RERANK_POOL = int(os.environ.get("RERANK_POOL", "50"))
 RERANK_TIMEOUT = float(os.environ.get("RERANK_TIMEOUT", "30"))
 
+# --- Reranker input budget (see docs-mcp-template + retrieval-lore.md) ------
+# jina-reranker-v2 is a BERT cross-encoder with bert.context_length=1024 and
+# LEARNED ABSOLUTE position embeddings: 1024 is a HARD ceiling, not something a
+# bigger llama.cpp --ubatch-size can lift. llama.cpp 500s the ENTIRE batch if
+# any (query, doc) pair exceeds it, so ONE oversized chunk silently drops that
+# whole query to fused order — the same ~90%->~62% P@1 cliff as the sidecar
+# being off the network, and just as quiet.
+#
+# A char cap is only a PROXY for tokens. Derive it from this corpus's measured
+# chars-per-token FLOOR (tokenise real chunks via {RERANK_URL}/tokenize and
+# take the min). Measured floor for this corpus: 1.47 (EPA/Bayer label text).
+RERANK_CTX_TOKENS = int(os.environ.get("RERANK_CTX_TOKENS", "1024"))
+RERANK_CHARS_PER_TOKEN = float(os.environ.get("RERANK_CHARS_PER_TOKEN", "1.45"))
+RERANK_QUERY_MAX_CHARS = int(os.environ.get("RERANK_QUERY_MAX_CHARS", "300"))
+# Margin covers [CLS]/[SEP] framing plus slack, because chars-per-token is an
+# ESTIMATE from a sample. Budget to ~94%, never to exactly 1024.
+_RERANK_MARGIN_TOKENS = int(os.environ.get("RERANK_MARGIN_TOKENS", "64"))
+_RERANK_QUERY_TOKENS = int(RERANK_QUERY_MAX_CHARS / RERANK_CHARS_PER_TOKEN) + 1
+RERANK_DOC_MAX_CHARS = max(
+    256,
+    int(
+        (RERANK_CTX_TOKENS - _RERANK_QUERY_TOKENS - _RERANK_MARGIN_TOKENS)
+        * RERANK_CHARS_PER_TOKEN
+    ),
+)
+
 HYBRID_SEARCH = os.environ.get("HYBRID_SEARCH", "").lower() in ("true", "1", "yes", "on")
 RRF_K = int(os.environ.get("RRF_K", "60"))
 
@@ -366,16 +392,20 @@ def _rerank_pool(
     pool: list[tuple[str, dict, float]],
 ) -> list[tuple[str, dict, float]]:
     """Send (query, doc_text) pairs to a llama.cpp /v1/rerank endpoint
-    and reorder by relevance score. Truncates docs to 2000 chars (the
-    jina-reranker GGUF rejects the ENTIRE batch if any pair exceeds
-    n_ctx_train=1024; full text still goes back to the user)."""
+    and reorder by relevance score. Both sides are truncated to the derived
+    RERANK_* budget above — the pair, not the doc alone, must fit the
+    reranker's 1024-token ceiling, and one oversized pair 500s the whole
+    batch. Truncation is for SCORING ONLY; full text still goes to the user."""
     import httpx
-    docs_truncated = [d[:2000] for d, _meta, _s in pool[:RERANK_POOL]]
+    docs_truncated = [
+        d[:RERANK_DOC_MAX_CHARS] for d, _meta, _s in pool[:RERANK_POOL]
+    ]
     if not docs_truncated:
         return pool
     r = httpx.post(
         f"{RERANK_URL}/v1/rerank",
-        json={"query": query, "documents": docs_truncated},
+        json={"query": query[:RERANK_QUERY_MAX_CHARS],
+              "documents": docs_truncated},
         timeout=RERANK_TIMEOUT,
     )
     r.raise_for_status()

rag/retrieval.py

@@ -25,6 +25,32 @@ BM25_DB = Path(os.environ.get("BM25_DB",
                str(REPO_ROOT / "bm25" / "crop_chem_docs.db")))
 COLLECTION = f"{os.environ.get('PRODUCT_NAME', 'crop_chem')}_docs"
 
+# --- Reranker input budget (see docs-mcp-template + retrieval-lore.md) ------
+# jina-reranker-v2 is a BERT cross-encoder with bert.context_length=1024 and
+# LEARNED ABSOLUTE position embeddings: 1024 is a HARD ceiling, not something a
+# bigger llama.cpp --ubatch-size can lift. llama.cpp 500s the ENTIRE batch if
+# any (query, doc) pair exceeds it, so ONE oversized chunk silently drops that
+# whole query to fused order — the same ~90%->~62% P@1 cliff as the sidecar
+# being off the network, and just as quiet.
+#
+# A char cap is only a PROXY for tokens. Derive it from this corpus's measured
+# chars-per-token FLOOR (tokenise real chunks via {RERANK_URL}/tokenize and
+# take the min). Measured floor for this corpus: 1.47 (EPA/Bayer label text).
+RERANK_CTX_TOKENS = int(os.environ.get("RERANK_CTX_TOKENS", "1024"))
+RERANK_CHARS_PER_TOKEN = float(os.environ.get("RERANK_CHARS_PER_TOKEN", "1.45"))
+RERANK_QUERY_MAX_CHARS = int(os.environ.get("RERANK_QUERY_MAX_CHARS", "300"))
+# Margin covers [CLS]/[SEP] framing plus slack, because chars-per-token is an
+# ESTIMATE from a sample. Budget to ~94%, never to exactly 1024.
+_RERANK_MARGIN_TOKENS = int(os.environ.get("RERANK_MARGIN_TOKENS", "64"))
+_RERANK_QUERY_TOKENS = int(RERANK_QUERY_MAX_CHARS / RERANK_CHARS_PER_TOKEN) + 1
+RERANK_DOC_MAX_CHARS = max(
+    256,
+    int(
+        (RERANK_CTX_TOKENS - _RERANK_QUERY_TOKENS - _RERANK_MARGIN_TOKENS)
+        * RERANK_CHARS_PER_TOKEN
+    ),
+)
+
 
 class Retriever(Protocol):
     name: str
@@ -230,7 +256,7 @@ class RerankedRetriever:
             if not d:
                 continue
             # Truncate to keep under the reranker's per-pair context limit
-            docs.append(d[:2000])
+            docs.append(d[:RERANK_DOC_MAX_CHARS])
             kept_pages.append((source, source_key))
 
         if not docs:
@@ -240,7 +266,8 @@ class RerankedRetriever:
         try:
             r = httpx.post(
                 f"{self.rerank_url}/v1/rerank",
-                json={"query": query, "documents": docs},
+                json={"query": query[:RERANK_QUERY_MAX_CHARS],
+                      "documents": docs},
                 timeout=self.timeout,
             )
             r.raise_for_status()