justin/hvm-docs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(chunker): MAX_CHARS 6000 → 4000 for table-dense content (#6)

Browse Source
This commit was merged in pull request #6.
This commit is contained in:
Justin Paul
2026-05-22 15:11:23 -04:00
parent faad4767c6
commit e07df7a1ae
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rag/chunk.py
+6 -2
View File
@@ -32,8 +32,12 @@ CHARS_PER_TOKEN = 4
TARGET_TOKENS = 500
TARGET_CHARS = TARGET_TOKENS * CHARS_PER_TOKEN
# Hard cap: nomic-embed-text's context is 2048 tokens. Anything larger
# 400s the entire embed batch. 6000 chars ≈ 1500 tokens leaves headroom.
MAX_CHARS = 6000
# 400s the entire embed batch. 6000 chars works for prose but markdown
# tables with lots of `|` separators tokenize ~1.4× denser; a 5839-char
# table chunk from the HVM qualification matrix tokenized past 2048 and
# crashed the rebuild. 4000 chars stays under 2048 tokens even for
# dense table content while leaving headroom for the query side.
MAX_CHARS = 4000
def _hard_split(text: str) -> list[str]: