name: Monthly corpus refresh # Runs the full pipeline: scrape all sources → rebuild indexes → # push image. Cron'd once a month (1st @ 06:00 UTC). Skip the # reindex + image-push if the scrape produced no diff against the # committed corpus. # # Runtime budget: act_runner kills the job container at exactly 3 h # (it runs as `/bin/sleep 10800`), which is what killed every run from # 2026-06-01 through 2026-09-01. The EPA step is incremental and # parallel so a steady-state refresh is ~15-20 min: cached not-row-crop # verdicts cost no request, and a product already on disk is only # re-downloaded when EPA reports a new label date. `timeout-minutes` # below fails the job legibly before the container disappears. on: schedule: - cron: "0 6 1 * *" # 1st of each month, 06:00 UTC workflow_dispatch: inputs: force_build: description: "Rebuild indexes + push image even if corpus is unchanged" type: boolean default: false sources: description: "Sources to scrape (comma-separated, blank = all)" type: string default: "" env: # Self-hosted Gitea registry on the same LAN as the runner. REGISTRY_PUSH: 192.168.0.2:1234 REGISTRY_PULL: git.jpaul.io IMAGE: ${{ github.repository }} # Embedder pool for the reindex step. Two Ollama instances on the # Gitea/runner host (one per GPU) + the Windows Ollama. Trashpanda's # Ollama is production-shared; CI doesn't hit it. OLLAMA_URL: http://192.168.0.2:11434,http://192.168.0.2:11435,http://192.168.0.125:11434 EMBED_MODEL: nomic-embed-text PRODUCT_NAME: crop_chem jobs: refresh: runs-on: docker # Below act_runner's own 3 h container lifetime, so an overrun fails # as a timeout instead of "container ... does not exist". timeout-minutes: 170 container: image: catthehacker/ubuntu:act-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Verify image name resolves # ${{ github.event.repository.name }} silently resolves to EMPTY on # schedule events (Gitea >=1.27), which built the tag # "192.168.0.2:1234//:latest" and failed the build only AFTER # the full scrape. IMAGE now comes from github.repository; this step # fails in seconds if it ever goes empty again. run: | echo "IMAGE=${IMAGE}" case "${IMAGE}" in */?*) ;; *) echo "ERROR: IMAGE did not resolve to owner/repo"; exit 1 ;; esac - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.12" - name: Install dependencies run: | python -m pip install -q --upgrade pip python -m pip install -q -r requirements.txt # ---- Phase 1: scrape --------------------------------------- - name: Scrape Bayer if: ${{ inputs.sources == '' || contains(inputs.sources, 'bayer') }} run: python -m scrape.runner --source bayer --force - name: Scrape EPA PPLS if: ${{ inputs.sources == '' || contains(inputs.sources, 'epa_ppls') }} # Deliberately NOT --force: that re-downloaded all ~11.4K candidate # registrations every month (~15-20 h of work) and never finished. # Without it the run is incremental — one cheap API call per product, # a PDF only when the label date actually moved — and the committed # filter cache skips the ~7.3K non-row-crop products outright. # Workers share one 5 req/sec ceiling, so this is faster without # being ruder. Local full re-fetch: add --force --no-filter-cache. run: python -m scrape.runner --source epa_ppls --workers 6 # ---- Commit corpus changes + retry-on-race ----------------- - name: Commit corpus changes (if any) id: commit run: | git config user.name "crop-chem-docs-refresh" git config user.email "actions@jpaul.io" # The filter-verdict cache is committed so next month starts warm, # but it changes on every run — only a real corpus diff may trigger # the reindex + image build, so `changed` is decided on the corpus # paths alone. git add sources.json corpus scrape/state if git diff --cached --quiet -- sources.json corpus; then echo "no corpus changes — skipping reindex and image build" echo "changed=false" >> "$GITHUB_OUTPUT" else echo "changed=true" >> "$GITHUB_OUTPUT" fi if git diff --cached --quiet; then echo "nothing staged — no commit" exit 0 fi ts=$(date -u +"%Y-%m-%dT%H:%MZ") n_bayer=$(find corpus/bayer -name '*.json' 2>/dev/null | wc -l) n_epa=$(find corpus/epa_ppls -name '*.json' 2>/dev/null | wc -l) git commit -m "monthly refresh: ${ts} — bayer=${n_bayer} epa_ppls=${n_epa}" attempt=1 while [ $attempt -le 3 ]; do if git push; then echo "pushed corpus changes (attempt $attempt)" break fi if [ $attempt -eq 3 ]; then echo "push still failing after 3 attempts"; exit 1 fi git fetch origin main git rebase origin/main || { echo "rebase conflict"; exit 1; } attempt=$((attempt + 1)) done # ---- Rebuild Chroma + BM25 --------------------------------- - name: Rebuild indexes if: steps.commit.outputs.changed == 'true' || inputs.force_build == true run: python -m rag.index --rebuild # ---- Build & push image ------------------------------------ - name: Log in to Gitea container registry if: steps.commit.outputs.changed == 'true' || inputs.force_build == true run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "${REGISTRY_PUSH}" -u "${{ github.repository_owner }}" --password-stdin - name: Build & push image if: steps.commit.outputs.changed == 'true' || inputs.force_build == true # Tags: :latest (Watchtower target), : (rollback pin), # :corpus- (links image to corpus version so # Drawbar can pin to a specific corpus snapshot). run: | SHA_TAG=$(echo "$GITHUB_SHA" | cut -c1-12) CORPUS_TAG="corpus-$(date -u +%Y.%m.%d)" docker build \ -t "${REGISTRY_PUSH}/${IMAGE}:latest" \ -t "${REGISTRY_PUSH}/${IMAGE}:${SHA_TAG}" \ -t "${REGISTRY_PUSH}/${IMAGE}:${CORPUS_TAG}" \ . # Smoke: the image must at least import its server module before we # publish it. mcp 2.x renamed mcp.server.fastmcp -> mcp.server.mcpserver, # so an unpinned dep produced a green build that crash-looped in prod. # Import is side-effect-free here (lazy singletons), so this needs no # Ollama/Chroma. docker run --rm --entrypoint python \ "${REGISTRY_PUSH}/${IMAGE}:latest" -c "import docs_mcp.server" docker push "${REGISTRY_PUSH}/${IMAGE}:latest" docker push "${REGISTRY_PUSH}/${IMAGE}:${SHA_TAG}" docker push "${REGISTRY_PUSH}/${IMAGE}:${CORPUS_TAG}" - name: Link container package to this repo if: steps.commit.outputs.changed == 'true' || inputs.force_build == true env: GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} run: | OWNER="${{ github.repository_owner }}" PKG="${GITHUB_REPOSITORY##*/}" BODY=$(mktemp) CODE=$(curl -sS -o "$BODY" -w "%{http_code}" -X POST \ -H "Authorization: token ${GITEA_TOKEN}" \ "https://${REGISTRY_PULL}/api/v1/packages/${OWNER}/container/${PKG}/-/link/${PKG}") echo "link http=$CODE body=$(cat "$BODY")" case "$CODE" in 201) echo "linked package to ${OWNER}/${PKG}" ;; 400) echo "already linked — ok" ;; *) echo "unexpected status $CODE"; exit 1 ;; esac - name: Prune old container versions # GC requires broader scope than REGISTRY_TOKEN's push perms # (HTTP 403 on /packages/.../versions). Non-critical housekeeping. # TODO: issue separate PAT with admin:package scope. if: steps.commit.outputs.changed == 'true' || inputs.force_build == true continue-on-error: true env: GITEA_TOKEN: ${{ secrets.REGISTRY_TOKEN }} run: | python scripts/registry_gc.py \ --owner "${{ github.repository_owner }}" \ --package "${GITHUB_REPOSITORY##*/}" \ --keep-days 180 \ --keep-latest 6