docker: production image + Gitea Actions for monthly refresh

Dockerfile: self-contained image with corpus + Chroma + BM25 baked
in. Drawbar's compose pulls + runs without volume mounts. Built from
sources.json (labels schema), PRODUCT_NAME=crop_chem by default,
HYBRID_SEARCH=true (always-on for production quality). RERANK_URL +
OLLAMA_URL get set at compose time.

.gitea/workflows/refresh.yml: monthly cron (1st @ 06:00 UTC) does
full scrape → reindex → image push. Scrapes Bayer (~30 min) +
EPA PPLS row-crop filtered (~7h). Skips reindex+push if no corpus
diff. Tags pushed: :latest, :<sha12>, :corpus-<YYYY.MM.DD>.

.gitea/workflows/image-only.yml: on-demand or auto on code-only
pushes to main (paths: docs_mcp/, rag/, scrape/, requirements.txt,
Dockerfile, sources.json). Reindexes from committed corpus, builds
image, pushes. ~10 min vs ~9h full refresh.

.gitignore: corpus/ now COMMITTED (4,159 labels, 265 MB of .md +
sidecars). Lets image-only.yml rebuild indexes without re-scraping.
chroma/ + bm25/ still gitignored (regenerable binary indexes).

.dockerignore: drops venv, eval results, PLAN/README/CLAUDE.md,
deploy/, .git/ — keeps the image lean. corpus + chroma + bm25
explicitly NOT in dockerignore (those go INTO the image).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.gitea/workflows/image-only.yml

@@ -1,34 +1,32 @@
 name: Image rebuild (skip scrape)
 
-# Fast path for code-only changes. Skips the scrape and goes straight to:
-# rebuild indexes (from corpus already committed on main) + image build
-# + push. Runtime is ~18 min vs ~40 min for the full refresh.
+# Fast path for code-only changes. Skips the scrape and goes straight
+# to: rebuild indexes (from corpus already committed on main) + image
+# build + push. Runtime ~10 min vs ~9 h for the full monthly refresh.
 #
 # Use when a PR only changes code/config — anything where the upstream
 # corpus hasn't moved but we want the new Python in the running image.
-#
-# IMPORTANT: fetch-depth: 0 is required for the digest-history step
-# to find commits to walk. Don't change to 1.
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs_mcp/**"
+      - "rag/**"
+      - "scrape/**"
+      - "requirements.txt"
+      - "Dockerfile"
+      - "sources.json"
 
 env:
-  REGISTRY_PUSH: <lan-host>:<port>
-  REGISTRY_PULL: <public-registry-hostname>
-  # Image name derives from the actual repo at runtime, so a clone
-  # doesn't need to find/replace anything. e.g. justin/my-product-docs.
-  # github.* context is Gitea Actions' inherited GitHub-Actions namespace
-  # — values come from the Gitea server, not github.com.
+  REGISTRY_PUSH: git.jpaul.io
+  REGISTRY_PULL: git.jpaul.io
   IMAGE: ${{ github.repository_owner }}/${{ github.event.repository.name }}
-  OLLAMA_URL: http://<gpu-host>:11434
+  OLLAMA_URL: http://192.168.0.2:11434,http://192.168.0.2:11435,http://192.168.0.125:11434
   EMBED_MODEL: nomic-embed-text
-  # PRODUCT_NAME defaults to the repo name so a clone works without
-  # editing. Override here if you want a different identifier (e.g.
-  # repo "my-product-docs" → PRODUCT_NAME "myproduct"). Used as the
-  # Chroma collection name, BM25 db filename, and MCP server name —
-  # see docs_mcp/server.py.
-  PRODUCT_NAME: ${{ github.event.repository.name }}
+  PRODUCT_NAME: crop_chem
 
 jobs:
   build:
@@ -39,8 +37,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          # Full history (not shallow) so the digest-history step can
-          # walk git log up to --history-days back.
           fetch-depth: 0
 
       - name: Set up Python
@@ -53,46 +49,34 @@ jobs:
           python -m pip install -q --upgrade pip
           python -m pip install -q -r requirements.txt
 
-      - name: Refresh digest history
-        # Cheap (a few seconds); doesn't touch corpus content.
-        # Without this step, a code-only deploy would ship an
-        # increasingly-stale digest history relative to git.
-        run: |
-          mkdir -p corpus/.digest
-          python -m scrape.changelog \
-              --history-out corpus/.digest/history.jsonl \
-              --history-days 120
-
       - name: Verify committed corpus is present
         run: |
           test -d corpus || { echo "ERROR: corpus/ missing on this ref"; exit 1; }
-          echo "corpus: $(du -sh corpus | cut -f1), $(find corpus -name '*.md' | wc -l) markdown files"
+          n_md=$(find corpus -name '*.md' | wc -l)
+          n_json=$(find corpus -name '*.json' | wc -l)
+          echo "corpus: $(du -sh corpus | cut -f1) on disk, ${n_md} .md / ${n_json} .json"
+          test "$n_md" -gt 100 || { echo "ERROR: corpus has fewer than 100 labels — was the rename committed?"; exit 1; }
 
-      - name: Rebuild indexes from existing corpus
+      - name: Rebuild indexes from committed corpus
         run: python -m rag.index --rebuild
 
-      - name: Log in to registry (LAN endpoint)
+      - name: Log in to Gitea container registry
         run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "${REGISTRY_PUSH}" -u "${{ github.repository_owner }}" --password-stdin
 
       - name: Build & push image
         run: |
           SHA_TAG=$(echo "$GITHUB_SHA" | cut -c1-12)
-          DATE_TAG=$(date -u +%Y.%m.%d)
+          CORPUS_TAG="corpus-$(date -u +%Y.%m.%d)"
           docker build \
             -t "${REGISTRY_PUSH}/${IMAGE}:latest" \
             -t "${REGISTRY_PUSH}/${IMAGE}:${SHA_TAG}" \
-            -t "${REGISTRY_PUSH}/${IMAGE}:${DATE_TAG}" \
+            -t "${REGISTRY_PUSH}/${IMAGE}:${CORPUS_TAG}" \
             .
           docker push "${REGISTRY_PUSH}/${IMAGE}:latest"
           docker push "${REGISTRY_PUSH}/${IMAGE}:${SHA_TAG}"
-          docker push "${REGISTRY_PUSH}/${IMAGE}:${DATE_TAG}"
+          docker push "${REGISTRY_PUSH}/${IMAGE}:${CORPUS_TAG}"
 
       - name: Link container package to this repo
-        # Gitea container packages are owned by a USER, not a repo —
-        # they don't auto-appear under the repo's Packages tab.
-        # This API call creates the association. One-time-effective:
-        # re-running returns 400 once linked, which we swallow.
-        # Endpoint requires Gitea 1.21+.
         env:
           GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
         run: |
@@ -105,7 +89,7 @@ jobs:
           echo "link http=$CODE  body=$(cat "$BODY")"
           case "$CODE" in
             201) echo "linked package to ${OWNER}/${PKG}" ;;
-            400) echo "already linked (re-link returns 400) — ok" ;;
+            400) echo "already linked — ok" ;;
             *)   echo "unexpected status $CODE"; exit 1 ;;
           esac
 
@@ -116,5 +100,5 @@ jobs:
           python scripts/registry_gc.py \
             --owner "${{ github.repository_owner }}" \
             --package "${{ github.event.repository.name }}" \
-            --keep-days 90 \
-            --keep-latest 5
+            --keep-days 180 \
+            --keep-latest 6

.gitea/workflows/refresh.yml

@@ -1,45 +1,43 @@
-name: Weekly docs refresh
+name: Monthly corpus refresh
 
-# Runs the full pipeline: scrape upstream → rebuild indexes → push
-# image. Cron'd weekly (Mondays). Skip the reindex + image-push if the
-# scrape produced no diff against the committed corpus.
+# Runs the full pipeline: scrape all sources → rebuild indexes →
+# push image. Cron'd once a month (1st @ 06:00 UTC). Skip the
+# reindex + image-push if the scrape produced no diff against the
+# committed corpus.
 #
-# IMPORTANT: actions/checkout@v4 fetch-depth: 0 is required because
-# the digest-history step walks git log up to --history-days back.
-# With a shallow checkout the history file ships empty.
+# Bayer takes ~30 min; EPA PPLS takes ~7 h with row-crop +
+# registrant filters. The whole monthly job is ~8-9 h end-to-end.
+# If that's too long for the runner you can:
+#   - Run just one source: workflow_dispatch with sources="bayer"
+#   - Limit EPA at the scraper: edit the step to add "--limit 5000"
 
 on:
   schedule:
-    - cron: "0 6 * * 1"     # Mondays 06:00 UTC
+    - cron: "0 6 1 * *"     # 1st of each month, 06:00 UTC
   workflow_dispatch:
     inputs:
       force_build:
         description: "Rebuild indexes + push image even if corpus is unchanged"
         type: boolean
         default: false
+      sources:
+        description: "Sources to scrape (comma-separated, blank = all)"
+        type: string
+        default: ""
 
 env:
-  # If your registry sits behind Cloudflare with its 100 MB body cap,
-  # use a LAN endpoint for pushes (bypasses CF) and the public hostname
-  # for pulls (response bodies aren't capped).
-  REGISTRY_PUSH: <lan-host>:<port>
-  REGISTRY_PULL: <public-registry-hostname>
-  # Image name derives from the actual repo at runtime, so a clone
-  # doesn't need to find/replace anything. e.g. justin/my-product-docs.
-  # github.* context is Gitea Actions' inherited GitHub-Actions namespace
-  # — values come from the Gitea server, not github.com.
+  # Self-hosted Gitea registry on the same LAN as the runner.
+  REGISTRY_PUSH: git.jpaul.io
+  REGISTRY_PULL: git.jpaul.io
   IMAGE: ${{ github.repository_owner }}/${{ github.event.repository.name }}
 
-  # Embedder. One URL per GPU; the indexer round-robins.
-  OLLAMA_URL: http://<gpu-host>:11434
+  # Embedder pool for the reindex step. Two Ollama instances on the
+  # Gitea/runner host (one per GPU) + the Windows Ollama. Trashpanda's
+  # Ollama is production-shared; CI doesn't hit it.
+  OLLAMA_URL: http://192.168.0.2:11434,http://192.168.0.2:11435,http://192.168.0.125:11434
   EMBED_MODEL: nomic-embed-text
 
-  # PRODUCT_NAME defaults to the repo name so a clone works without
-  # editing. Override here if you want a different identifier (e.g.
-  # repo "my-product-docs" → PRODUCT_NAME "myproduct"). Used as the
-  # Chroma collection name, BM25 db filename, and MCP server name —
-  # see docs_mcp/server.py.
-  PRODUCT_NAME: ${{ github.event.repository.name }}
+  PRODUCT_NAME: crop_chem
 
 jobs:
   refresh:
@@ -50,9 +48,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          # Full history — required for the digest-history step to
-          # walk git log. Default fetch-depth: 1 silently produces a
-          # 0-byte history file.
           fetch-depth: 0
 
       - name: Set up Python
@@ -66,52 +61,33 @@ jobs:
           python -m pip install -q -r requirements.txt
 
       # ---- Phase 1: scrape ---------------------------------------
-      - name: Refresh bundle catalog
-        run: python -m scrape.bundles
+      - name: Scrape Bayer
+        if: ${{ inputs.sources == '' || contains(inputs.sources, 'bayer') }}
+        run: python -m scrape.runner --source bayer --force
 
-      - name: Re-scrape all bundles
-        # --force re-fetches every page so we actually see upstream
-        # edits. Without it the runner skips pages already on disk.
-        run: python -m scrape.runner --all --force --concurrency 6
+      - name: Scrape EPA PPLS
+        if: ${{ inputs.sources == '' || contains(inputs.sources, 'epa_ppls') }}
+        # Row-crop + registrant filters keep this to ~16K PDFs / ~7h.
+        # Pass --no-row-crop-filter or --no-registrant-filter to broaden.
+        run: python -m scrape.runner --source epa_ppls --force
 
-      # ---- Build the digest history BEFORE committing ------------
-      # See PLAN.md Phase 13. Walks recent corpus-touching commits
-      # and writes corpus/.digest/history.jsonl. The current refresh
-      # gets added on the NEXT run's history (one-week lag is fine).
-      - name: Build digest history
-        run: |
-          mkdir -p corpus/.digest
-          python -m scrape.changelog \
-              --history-out corpus/.digest/history.jsonl \
-              --history-days 120
-
-      # ---- Commit + retry-on-race --------------------------------
+      # ---- Commit corpus changes + retry-on-race -----------------
       - name: Commit corpus changes (if any)
         id: commit
         run: |
-          git config user.name "<product>-docs-refresh"
-          git config user.email "actions@<your-domain>"
-          git add bundles.json corpus
+          git config user.name "crop-chem-docs-refresh"
+          git config user.email "actions@jpaul.io"
+          git add sources.json corpus
           if git diff --cached --quiet; then
             echo "no corpus changes — skipping reindex and image build"
             echo "changed=false" >> "$GITHUB_OUTPUT"
             exit 0
           fi
           echo "changed=true" >> "$GITHUB_OUTPUT"
-          python -m scrape.changelog --cached > /tmp/changelog.txt
-          summary=$(head -1 /tmp/changelog.txt)
           ts=$(date -u +"%Y-%m-%dT%H:%MZ")
-          {
-            echo "weekly refresh: ${ts} — ${summary}"
-            echo ""
-            cat /tmp/changelog.txt
-          } > /tmp/commitmsg.txt
-          git commit -F /tmp/commitmsg.txt
-          # Retry on race: if main moved while we were scraping (a
-          # human merged a PR during the run), `git push` rejects
-          # with "fetch first". Rebase our corpus commit onto new
-          # main and retry. Corpus + code paths are disjoint, so
-          # the rebase is trivially clean.
+          n_bayer=$(find corpus/bayer -name '*.json' 2>/dev/null | wc -l)
+          n_epa=$(find corpus/epa_ppls -name '*.json' 2>/dev/null | wc -l)
+          git commit -m "monthly refresh: ${ts} — bayer=${n_bayer} epa_ppls=${n_epa}"
           attempt=1
           while [ $attempt -le 3 ]; do
             if git push; then
@@ -119,47 +95,41 @@ jobs:
               break
             fi
             if [ $attempt -eq 3 ]; then
-              echo "push still failing after 3 attempts — bailing"
-              exit 1
+              echo "push still failing after 3 attempts"; exit 1
             fi
             git fetch origin main
-            git rebase origin/main || { echo "rebase conflict — bailing"; exit 1; }
+            git rebase origin/main || { echo "rebase conflict"; exit 1; }
             attempt=$((attempt + 1))
           done
 
-      # ---- Reindex Chroma + BM25 ---------------------------------
+      # ---- Rebuild Chroma + BM25 ---------------------------------
       - name: Rebuild indexes
         if: steps.commit.outputs.changed == 'true' || inputs.force_build == true
         run: python -m rag.index --rebuild
 
       # ---- Build & push image ------------------------------------
-      - name: Log in to registry (LAN endpoint)
+      - name: Log in to Gitea container registry
         if: steps.commit.outputs.changed == 'true' || inputs.force_build == true
         run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "${REGISTRY_PUSH}" -u "${{ github.repository_owner }}" --password-stdin
 
       - name: Build & push image
         if: steps.commit.outputs.changed == 'true' || inputs.force_build == true
-        # Runner shell is /bin/sh — use cut instead of ${VAR::N}.
-        # Three tags: :latest (Watchtower target), :<sha12>
-        # (rollback pin), :<YYYY.MM.DD> (human-readable).
+        # Tags: :latest (Watchtower target), :<sha12> (rollback pin),
+        # :corpus-<YYYY.MM.DD> (links image to corpus version so
+        # Drawbar can pin to a specific corpus snapshot).
         run: |
           SHA_TAG=$(echo "$GITHUB_SHA" | cut -c1-12)
-          DATE_TAG=$(date -u +%Y.%m.%d)
+          CORPUS_TAG="corpus-$(date -u +%Y.%m.%d)"
           docker build \
             -t "${REGISTRY_PUSH}/${IMAGE}:latest" \
             -t "${REGISTRY_PUSH}/${IMAGE}:${SHA_TAG}" \
-            -t "${REGISTRY_PUSH}/${IMAGE}:${DATE_TAG}" \
+            -t "${REGISTRY_PUSH}/${IMAGE}:${CORPUS_TAG}" \
             .
           docker push "${REGISTRY_PUSH}/${IMAGE}:latest"
           docker push "${REGISTRY_PUSH}/${IMAGE}:${SHA_TAG}"
-          docker push "${REGISTRY_PUSH}/${IMAGE}:${DATE_TAG}"
+          docker push "${REGISTRY_PUSH}/${IMAGE}:${CORPUS_TAG}"
 
       - name: Link container package to this repo
-        # Gitea container packages are owned by a USER, not a repo —
-        # they don't auto-appear under the repo's Packages tab.
-        # This API call creates the association. One-time-effective:
-        # re-running returns 400 once linked, which we swallow.
-        # Endpoint requires Gitea 1.21+.
         if: steps.commit.outputs.changed == 'true' || inputs.force_build == true
         env:
           GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
@@ -173,11 +143,10 @@ jobs:
           echo "link http=$CODE  body=$(cat "$BODY")"
           case "$CODE" in
             201) echo "linked package to ${OWNER}/${PKG}" ;;
-            400) echo "already linked (re-link returns 400) — ok" ;;
+            400) echo "already linked — ok" ;;
             *)   echo "unexpected status $CODE"; exit 1 ;;
           esac
 
-      # ---- Registry GC -------------------------------------------
       - name: Prune old container versions
         if: steps.commit.outputs.changed == 'true' || inputs.force_build == true
         env:
@@ -186,5 +155,5 @@ jobs:
           python scripts/registry_gc.py \
             --owner "${{ github.repository_owner }}" \
             --package "${{ github.event.repository.name }}" \
-            --keep-days 90 \
-            --keep-latest 5
+            --keep-days 180 \
+            --keep-latest 6