From 94fe8d9b54db7fd78579d7058d595339e31e95e2 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 16 Oct 2026 13:57:06 +0530 Subject: [PATCH 1/4] fix(documents): keep header fields, use PDF tables, survive cut-off answers --- .../classes/DocumentInterpreter.ts | 491 ++++++++++++++++-- 1 file changed, 452 insertions(+), 39 deletions(-) diff --git a/server/src/workflow-management/classes/DocumentInterpreter.ts b/server/src/workflow-management/classes/DocumentInterpreter.ts index 8632e398..c72ddde4 100644 --- a/server/src/workflow-management/classes/DocumentInterpreter.ts +++ b/server/src/workflow-management/classes/DocumentInterpreter.ts @@ -16,9 +16,22 @@ import * as XLSX from 'xlsx'; const MAX_SCHEMA_SAMPLE_CHARS = 16000; const MAX_CHUNK_CHARS = 12000; +/** + * Chunk size when the schema collects records (an array of objects). The JSON + * for a table is about as long as the table, and the models stop at ~4k + * output tokens (~12k chars), so 12k-char chunks of rows were routinely cut + * off. Smaller chunks fit in one answer instead of being redone in halves. + */ +const MAX_RECORD_CHUNK_CHARS = 6000; +/** Target answer size per chunk, under the ~12k-char (4k-token) output cap. */ +const TARGET_OUTPUT_CHARS = 9000; +const MIN_CHUNK_CHARS = 2500; const MAX_FILE_SIZE_MB = 10; const MAX_TABLES_IN_CONTEXT = 5; const SCANNED_THRESHOLD_CHARS_PER_PAGE = 30; +const MAX_TABLE_DETECTION_PAGES = 200; +const EXTRACTION_CONCURRENCY = 3; +const MAX_TRUNCATION_SPLITS = 3; export const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024; @@ -42,19 +55,29 @@ interface ExtractionSchemaField { type ExtractionSchema = Record; +/** Page content in reading order: running text and tables, kept apart so tables keep their columns. */ +type ContentBlock = { type: 'text'; text: string } | { type: 'table'; rows: string[][] }; + interface ParsedPage { pageNumber: number; + /** Markdown-ish text of the page; tables are rendered as Markdown tables. */ text: string; + /** Sheet name for spreadsheets. */ + label?: string; + blocks?: ContentBlock[]; } interface ParsedDocument { text: string; pageCount: number; pages: ParsedPage[]; + /** Document-level tables that aren't tied to a page block (OCR key/value tables). */ tables: string[][][]; sourceHtml?: string; /** True when the document came from a flat image, which has no page structure. */ isImage?: boolean; + /** XLSX/CSV: every page is one table. */ + isSpreadsheet?: boolean; } interface ExtractionResult { @@ -76,6 +99,8 @@ export interface ParsedOutput { interface ProviderResult { payload: any; provider: string; + /** The model hit its output limit; the payload (if any) was salvaged from cut-off JSON. */ + truncated?: boolean; } const normalizeWhitespace = (value: string): string => @@ -84,13 +109,119 @@ const normalizeWhitespace = (value: string): string => const cleanText = (value: string): string => normalizeWhitespace(value.replace(/\x00/g, '')); +const escapeTableCell = (cell: string): string => + String(cell ?? '').replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').replace(/\s+/g, ' ').trim(); + +/** Markdown table from rows; the first row is the header. Ragged rows are padded. */ +const renderMarkdownTable = (rows: string[][]): string => { + const width = Math.max(1, ...rows.map((row) => row.length)); + const cells = (row: string[]) => Array.from({ length: width }, (_, i) => escapeTableCell(row[i] ?? '') || ' '); + const [head, ...body] = rows; + return [ + `| ${cells(head || []).join(' | ')} |`, + `| ${Array(width).fill('---').join(' | ')} |`, + ...body.map((row) => `| ${cells(row).join(' | ')} |`), + ].join('\n'); +}; + const escapeHtml = (value: string): string => - value - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); + String(value ?? '').replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + +const renderHtmlTable = (rows: string[][]): string => { + const [head, ...body] = rows; + const headHtml = `${(head || []).map((cell) => `${escapeHtml(cell.trim())}`).join('')}`; + const bodyHtml = `${body.map((row) => `${row.map((cell) => `${escapeHtml(cell.trim())}`).join('')}`).join('\n')}`; + return `\n${headHtml}\n${bodyHtml}\n
`; +}; + +/** + * Word tables rarely mark a header row, so HTML→Markdown gives them an empty + * header (`| | |` + separator) above the real first row. Promote that row. + */ +const promoteEmptyTableHeaders = (markdown: string): string => + markdown.replace( + /^\|(?:[ \t]*\|)+[ \t]*\n(\|(?:[ \t]*:?-{3,}:?[ \t]*\|)+)[ \t]*\n(\|.*\|)[ \t]*$/gm, + '$2\n$1' + ); + +const renderBlocksMarkdown = (blocks: ContentBlock[]): string => + blocks + .map((block) => (block.type === 'text' ? block.text : renderMarkdownTable(block.rows))) + .filter((part) => part.trim()) + .join('\n\n'); + +const renderBlocksHtml = (blocks: ContentBlock[]): string => + blocks + .map((block) => + block.type === 'text' + ? block.text.split(/\n{2,}/).map((p) => `

${escapeHtml(p).replace(/\n/g, '
')}

`).join('\n') + : renderHtmlTable(block.rows) + ) + .join('\n'); + +/** Trim every line (PDF text lines carry padding) and collapse blank runs. */ +const tidyLines = (value: string): string => + cleanText(value) + .split('\n') + .map((line) => line.trim()) + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + +const normalizeCellForMatch = (cell: string): string => cell.replace(/\s+/g, ' ').trim().toLowerCase(); + +/** + * Lays out a PDF page as text and tables in reading order. + * + * pdf-parse returns the page text with a table's cells separated only by + * runs of spaces, which cleanText then collapsed into single spaces, so + * "Widget B1 | 6 | $379.41" reached the model as "Widget B1 6 $379.41". The + * detected tables are matched back to their text lines (a line whose + * space-separated cells equal a table row) and replace them in place, so + * each table appears once, with its columns, where it sits on the page. + * A table whose rows can't be found in the text is appended to the page. + */ +const buildPdfPageBlocks = (rawText: string, tables: string[][][]): ContentBlock[] => { + const signature = (cells: string[]) => { + const parts = cells.map(normalizeCellForMatch).filter(Boolean); + return parts.length >= 2 ? parts.join('\u0001') : ''; + }; + const rowOwner = new Map(); + tables.forEach((table, tableIndex) => { + table.forEach((row) => { + const sig = signature(row); + if (sig && !rowOwner.has(sig)) rowOwner.set(sig, tableIndex); + }); + }); + + const blocks: ContentBlock[] = []; + const emitted = new Set(); + let textLines: string[] = []; + const flushText = () => { + const text = tidyLines(textLines.join('\n')); + if (text) blocks.push({ type: 'text', text }); + textLines = []; + }; + + for (const line of rawText.replace(/\r/g, '').replace(/\u0000/g, '').split('\n')) { + const owner = rowOwner.get(signature(line.trim().split(/\t|\s{2,}/))); + if (owner === undefined) { + textLines.push(line); + continue; + } + if (!emitted.has(owner)) { + flushText(); + blocks.push({ type: 'table', rows: tables[owner] }); + emitted.add(owner); + } + } + flushText(); + + tables.forEach((table, tableIndex) => { + if (!emitted.has(tableIndex) && table.length > 0) blocks.push({ type: 'table', rows: table }); + }); + return blocks; +}; const OCR_ASCII_NOISE_THRESHOLD = 0.80; @@ -346,9 +477,58 @@ const extractBalancedJson = (value: string): string | null => { if (depth === 0) return value.slice(start, i + 1); } } + // The outermost value never closed: the output was cut off. Don't fall + // through to the first complete *inner* object, which silently turned a + // truncated 100-item list into a single item. + return null; + } + return null; +}; + +/** + * Recovers the complete part of JSON that was cut off mid-output: drops the + * partial trailing element and closes the open brackets. + * `{"items":[{"a":1},{"a":2` → `{"items":[{"a":1}]}`. + */ +const salvageTruncatedJson = (value: string): any | null => { + const start = value.search(/[{[]/); + if (start < 0) return null; + const text = value.slice(start); + const stack: string[] = []; + let inString = false; + let escaped = false; + let lastCut = -1; + let stackAtCut: string[] = []; + + for (let i = 0; i < text.length; i++) { + const char = text[i]; + if (inString) { + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === '{') stack.push('}'); + else if (char === '[') stack.push(']'); + else if (char === '}' || char === ']') { + stack.pop(); + if (stack.length === 0) return null; // complete, not truncated + lastCut = i + 1; + stackAtCut = [...stack]; + } else if (char === ',') { + lastCut = i; + stackAtCut = [...stack]; + } } - return null; + if (lastCut < 0) return null; + const candidate = text.slice(0, lastCut).replace(/,\s*$/, '') + [...stackAtCut].reverse().join(''); + try { + return JSON.parse(repairJsonText(candidate)); + } catch { + return null; + } }; const extractJsonText = (value: string): string => { @@ -372,6 +552,22 @@ const parseJsonLoose = (value: string): any => { } }; +/** Parse model output; salvage the complete part when it was cut off. */ +const parseModelJson = (content: string, hitOutputLimit: boolean): { payload: any; truncated: boolean } => { + const cleaned = stripCodeFences(stripReasoning(content)); + if (hitOutputLimit) { + const salvaged = salvageTruncatedJson(cleaned); + if (salvaged !== null) return { payload: salvaged, truncated: true }; + } + try { + return { payload: parseJsonLoose(content), truncated: hitOutputLimit }; + } catch (error) { + const salvaged = salvageTruncatedJson(cleaned); + if (salvaged !== null) return { payload: salvaged, truncated: true }; + throw error; + } +}; + const isMeaningful = (value: any): boolean => { if (value === null || value === undefined) return false; if (typeof value === 'string') return value.trim().length > 0; @@ -380,13 +576,57 @@ const isMeaningful = (value: any): boolean => { return true; }; +/** + * Numbers as written in documents: currency symbols and codes, thousands + * separators in western (1,234,567.89), European (1.234.567,89) and Indian + * (1,23,456) styles, accounting negatives "(1,200.00)" and trailing minus. + * The old version stripped everything but digits, dots and minus, so + * "1.234,56" became 1.23456 and "(500)" became +500. Ranges ("10-20") give null. + */ const parseNumericValue = (value: any): number | null => { if (typeof value === 'number' && Number.isFinite(value)) return value; if (typeof value !== 'string') return null; - const cleaned = value.replace(/[^0-9.-]+/g, ''); - if (!cleaned || cleaned === '-' || cleaned === '.') return null; + const raw = value.trim(); + if (!raw) return null; + + const negative = /^\(.*\)$/.test(raw) || /^[^\d]*-/.test(raw) || /-\s*$/.test(raw); + let cleaned = raw.replace(/[^0-9.,-]/g, ''); + if (/\d-\d/.test(cleaned)) return null; + cleaned = cleaned.replace(/-/g, ''); + if (!/\d/.test(cleaned)) return null; + + const lastComma = cleaned.lastIndexOf(','); + const lastDot = cleaned.lastIndexOf('.'); + if (lastComma > -1 && lastDot > -1) { + cleaned = lastComma > lastDot + ? cleaned.replace(/\./g, '').replace(',', '.') + : cleaned.replace(/,/g, ''); + } else if (lastComma > -1) { + const parts = cleaned.split(','); + // One comma followed by other than 3 digits is a decimal comma ("12,5"). + cleaned = parts.length === 2 && parts[1].length !== 3 ? parts.join('.') : parts.join(''); + } else if ((cleaned.match(/\./g) || []).length > 1) { + cleaned = cleaned.replace(/\./g, ''); + } const parsed = Number(cleaned); - return Number.isFinite(parsed) ? parsed : null; + if (!Number.isFinite(parsed)) return null; + return negative ? -Math.abs(parsed) : parsed; +}; + +const pad2 = (n: number) => String(n).padStart(2, '0'); + +/** + * ISO date for a date as written. Date-only values stay dates: "March 15, + * 2024" used to go through toISOString(), which on a server east of UTC + * gave "2024-03-14T18:30:00.000Z", a day early. + */ +const normalizeDateValue = (value: string): string => { + const normalized = value.trim(); + if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) return normalized; + const parsed = new Date(normalized); + if (Number.isNaN(parsed.getTime()) || !/\d/.test(normalized)) return normalized; + if (/\d{1,2}:\d{2}/.test(normalized)) return parsed.toISOString(); + return `${parsed.getFullYear()}-${pad2(parsed.getMonth() + 1)}-${pad2(parsed.getDate())}`; }; const parseBooleanValue = (value: any): boolean | null => { @@ -433,22 +673,13 @@ const sanitizeSchemaField = (input: any, fallbackLabel: string): ExtractionSchem return field; }; -const isRecordArray = (value: any): boolean => - Array.isArray(value) && - value.length > 0 && - value.every((item) => item !== null && typeof item === 'object' && !Array.isArray(item)); - -const normalizeExtractionData = (data: any): any => { - if (!data || typeof data !== 'object' || Array.isArray(data)) return data; - - const listEntries = Object.entries(data).filter(([, value]) => isRecordArray(value)); - if (listEntries.length === 0) return data; - - return listEntries.reduce>((acc, [key, value]) => { - acc[key] = value; - return acc; - }, {}); -}; +/** + * Final result shape. This used to keep only the list fields whenever the + * result had any, so an invoice came back as its line items without the + * invoice number, date, vendor or totals. The run view, API and integrations + * all handle scalar fields next to lists, so the result is kept whole. + */ +const normalizeExtractionData = (data: any): any => data; const sanitizeSchema = (input: any, prompt?: string): ExtractionSchema => { const candidate = @@ -547,8 +778,7 @@ const normalizeValueBySchema = (value: any, field: ExtractionSchemaField): any = case 'boolean': return parseBooleanValue(value); case 'date': { if (typeof value !== 'string') return null; - const parsed = new Date(value.trim()); - return Number.isNaN(parsed.getTime()) ? value.trim() : parsed.toISOString(); + return normalizeDateValue(value) || null; } case 'array': { const items = Array.isArray(value) @@ -580,23 +810,96 @@ const normalizeValueBySchema = (value: any, field: ExtractionSchemaField): any = return typeof value === 'string' ? cleanText(value) : String(value); } }; + /** + * Models often wrap the answer ({"data": {...}}, {"result": {...}}, + * {"extracted": {...}}). Unwrap single-key envelopes until the schema keys + * show up. + */ +const unwrapPayload = (payload: any, schema: ExtractionSchema): any => { + const schemaKeys = Object.keys(schema).map(toFieldKey); + let current = payload; + for (let depth = 0; depth < 3; depth++) { + if (!current || typeof current !== 'object' || Array.isArray(current)) break; + const keys = Object.keys(current); + if (keys.some((key) => schemaKeys.includes(toFieldKey(key)))) return current; + if (keys.length !== 1 || !current[keys[0]] || typeof current[keys[0]] !== 'object') break; + current = current[keys[0]]; + } + return payload; +}; -const normalizeResultToSchema = (raw: any, schema: ExtractionSchema): Record => - Object.entries(schema).reduce>((acc, [key, field]) => { - const sourceValue = raw?.[key] ?? raw?.[field.label] ?? raw?.[field.label.toLowerCase()] ?? null; +const normalizeResultToSchema = (raw: any, schema: ExtractionSchema): Record => { + // Match keys loosely too: "Invoice Number", "invoiceNumber", "invoice-number". + const byLooseKey = new Map(); + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + for (const [key, value] of Object.entries(raw)) { + byLooseKey.set(toFieldKey(key.replace(/([a-z])([A-Z])/g, '$1_$2')), value); + } + } + return Object.entries(schema).reduce>((acc, [key, field]) => { + const sourceValue = + raw?.[key] ?? + raw?.[field.label] ?? + raw?.[field.label.toLowerCase()] ?? + byLooseKey.get(key) ?? + byLooseKey.get(toFieldKey(field.label)) ?? + null; acc[key] = normalizeValueBySchema(sourceValue, field); return acc; }, {}); +}; +/** Comparison key for an extracted item: case, spacing and number formatting don't make a new item. */ +const itemKey = (item: any): string => { + const norm = (value: any): any => { + if (value === null || value === undefined) return null; + if (typeof value === 'number') return value; + if (typeof value === 'string') { + const numeric = /\d/.test(value) && /^[\s$€£₹¥(+-]*[\d.,\s]+[)%]?\s*$/.test(value) ? parseNumericValue(value) : null; + return numeric !== null ? numeric : value.replace(/\s+/g, ' ').trim().toLowerCase(); + } + if (Array.isArray(value)) return value.map(norm); + if (typeof value === 'object') { + return Object.keys(value).sort().reduce>((acc, key) => { + acc[key] = norm(value[key]); + return acc; + }, {}); + } + return value; + }; + return JSON.stringify(norm(item)); +}; + +/** + * Chunks are merged in document order. + * - Scalars: the first chunk that has the value wins (headers, parties and + * dates sit early). Previously the longest string from any chunk won, + * which let a later chunk's rambling or wrong value replace a correct one. + * - Arrays: items are appended in order. An item already collected from an + * earlier chunk isn't added again, but genuinely repeated items within one + * chunk (two identical line items) are kept. + */ const mergeValues = (current: any, incoming: any, field: ExtractionSchemaField): any => { if (!isMeaningful(current)) return incoming; if (!isMeaningful(incoming)) return current; if (field.type === 'array') { - const combined = [...(Array.isArray(current) ? current : []), ...(Array.isArray(incoming) ? incoming : [])]; - return combined.filter((item, index) => - combined.findIndex((c) => JSON.stringify(c) === JSON.stringify(item)) === index - ); + const base = Array.isArray(current) ? current : []; + const additions = Array.isArray(incoming) ? incoming : []; + const collected = new Map(); + base.forEach((item) => { + const key = itemKey(item); + collected.set(key, (collected.get(key) ?? 0) + 1); + }); + const seenInChunk = new Map(); + const merged = [...base]; + for (const item of additions) { + const key = itemKey(item); + const n = (seenInChunk.get(key) ?? 0) + 1; + seenInChunk.set(key, n); + if (n > (collected.get(key) ?? 0)) merged.push(item); + } + return merged; } if (field.type === 'object' && field.fields) { @@ -606,8 +909,7 @@ const mergeValues = (current: any, incoming: any, field: ExtractionSchemaField): }, {}); } - if (field.type === 'string') return String(incoming).length > String(current).length ? incoming : current; - return current ?? incoming; + return current; }; const mergeExtractionResults = ( @@ -644,7 +946,29 @@ const fallbackExtract = (text: string, schema: ExtractionSchema): Record => { +const TABLE_ROW = /^\|.*\|\s*$/; +const TABLE_SEPARATOR = /^\|(\s*:?-{3,}:?\s*\|)+\s*$/; + +/** + * When a Markdown table is split across segments, repeat its header row at + * the top of the next segment. Without it a later chunk of a spreadsheet or + * a long PDF table reached the model as bare rows with no column names. + */ +const carryTableHeaders = (segments: string[]): string[] => { + let header: string[] | null = null; + return segments.map((segment) => { + const lines = segment.split('\n'); + const continuesTable = header && TABLE_ROW.test(lines[0] || '') && !TABLE_SEPARATOR.test(lines[1] || ''); + const out = continuesTable ? [...header!, ...lines] : lines; + for (let i = 0; i < lines.length; i++) { + if (TABLE_SEPARATOR.test(lines[i]) && i > 0 && TABLE_ROW.test(lines[i - 1])) header = [lines[i - 1], lines[i]]; + else if (!TABLE_ROW.test(lines[i])) header = lines[i].trim() ? null : header; + } + return out.join('\n'); + }); +}; + +const buildChunks = (pages: ParsedPage[], budget: number = MAX_CHUNK_CHARS): Array<{ pageRange: string; text: string }> => { const chunks: Array<{ pageRange: string; text: string }> = []; let currentPages: ParsedPage[] = []; let currentLength = 0; @@ -656,7 +980,7 @@ const buildChunks = (pages: ParsedPage[]): Array<{ pageRange: string; text: stri const pageLength = pageText.length; if (currentPages.length > 0 && currentLength + pageLength > MAX_CHUNK_CHARS) { chunks.push({ - pageRange: `${currentPages[0].pageNumber}-${currentPages[currentPages.length - 1].pageNumber}`, + pageRange: `${currentPages[0].pageNumber} to ${currentPages[currentPages.length - 1].pageNumber}`, text: currentPages.map((p) => `Page ${p.pageNumber}\n${cleanText(p.text)}`).join('\n\n'), }); currentPages = []; @@ -677,6 +1001,95 @@ const buildChunks = (pages: ParsedPage[]): Array<{ pageRange: string; text: stri return chunks.length > 0 ? chunks : [{ pageRange: '1-1', text: '' }]; }; +type DocumentChunk = { pageRange: string; text: string }; + +/** + * Halves a chunk whose extraction was cut off by the model's output limit, + * at a blank line near the middle where possible; a table split in two keeps + * its header in both halves. + */ +const splitChunkInTwo = (chunk: DocumentChunk): [DocumentChunk, DocumentChunk] | null => { + if (chunk.text.length < 1500) return null; + const lines = chunk.text.split('\n'); + if (lines.length < 4) return null; + const mid = Math.floor(lines.length / 2); + let cut = mid; + for (let d = 0; d < lines.length / 4; d++) { + if (lines[mid + d] !== undefined && !lines[mid + d].trim()) { cut = mid + d; break; } + if (lines[mid - d] !== undefined && !lines[mid - d].trim()) { cut = mid - d; break; } + } + const [first, second] = carryTableHeaders([lines.slice(0, cut).join('\n'), lines.slice(cut).join('\n')]); + if (!first.trim() || !second.trim()) return null; + return [ + { pageRange: `${chunk.pageRange} (first half)`, text: first }, + { pageRange: `${chunk.pageRange} (second half)`, text: second }, + ]; +}; + +const mapWithConcurrency = async (items: T[], limit: number, fn: (item: T, index: number) => Promise): Promise => { + const results: R[] = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await fn(items[index], index); + } + }); + await Promise.all(workers); + return results; +}; + +/** The JSON shape the model should return, with placeholder types. */ +const schemaSkeleton = (schema: ExtractionSchema): Record => + Object.entries(schema).reduce>((acc, [key, field]) => { + if (field.type === 'array') { + const items = field.items; + acc[key] = [items?.type === 'object' && items.fields ? schemaSkeleton(items.fields) : `${items?.type || 'string'}`]; + } else if (field.type === 'object' && field.fields) { + acc[key] = schemaSkeleton(field.fields); + } else { + acc[key] = `${field.type} or null`; + } + return acc; + }, {}); + +/** + * Input chunk size for record extraction, from the document's own tables: + * each table row comes back as a JSON object holding the same values plus + * every field name and quoting, so narrow rows with long field names produce + * output twice the size of the input. + */ +const recordChunkBudget = (pages: ParsedPage[], schema: ExtractionSchema): number => { + const itemFields = Object.values(schema) + .filter((field) => field.type === 'array' && field.items?.type === 'object' && field.items.fields) + .map((field) => Object.keys(field.items!.fields!)); + if (itemFields.length === 0) return MAX_RECORD_CHUNK_CHARS; + const overheadPerItem = Math.max(...itemFields.map((keys) => keys.reduce((sum, key) => sum + key.length + 9, 4))); + + let rowChars = 0; + let rowCount = 0; + for (const page of pages) { + for (const block of page.blocks || []) { + if (block.type !== 'table') continue; + for (const row of block.rows.slice(1)) { + rowChars += row.reduce((sum, cell) => sum + String(cell).length + 3, 2); + rowCount++; + } + } + } + if (rowCount === 0) return MAX_RECORD_CHUNK_CHARS; + + const avgRow = rowChars / rowCount; + const rowsPerChunk = TARGET_OUTPUT_CHARS / (avgRow + overheadPerItem); + return Math.round(Math.min(MAX_CHUNK_CHARS, Math.max(MIN_CHUNK_CHARS, rowsPerChunk * avgRow))); +}; + +const scalarsOnly = (data: Record, schema: ExtractionSchema): Record => + Object.entries(schema).reduce>((acc, [key, field]) => { + acc[key] = field.type === 'array' ? [] : data[key]; + return acc; + }, {}); + const PADDLE_MODEL_BASE = 'https://media.githubusercontent.com/media/PT-Perkasa-Pilar-Utama/ppu-paddle-ocr-models/main'; const PADDLE_DICT_BASE = 'https://raw.githubusercontent.com/PT-Perkasa-Pilar-Utama/ppu-paddle-ocr-models/main'; const PADDLE_DETECTION = `${PADDLE_MODEL_BASE}/detection/PP-OCRv5_mobile_det_infer.onnx`; From e390d9c620604923d469b652929b25ba2277b730 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:34:47 +0000 Subject: [PATCH 2/4] fix(documents): wire table-aware parsing and cut-off recovery into extraction Completes the port of the document extraction fixes: the helpers were added but not used yet. - Chunking: split oversized pages (CSV/DOCX are one page), repeat table headers across chunks, label sheets, size record chunks from the document's own table rows. - Providers: report hitting the output limit (Ollama done_reason, OpenAI finish_reason, Anthropic stop_reason) and salvage the complete JSON. - Extraction: chunks run 3 at a time, unwrap envelopes, re-extract cut-off chunks in halves (up to 3 levels), tables only as context for single-chunk documents, part N of M prompts with the JSON shape. - PDF: per-page tables from pdf-parse laid out in place. - XLSX/CSV: one Markdown table per sheet. DOCX: Markdown with tables. - Markdown/HTML output rendered from page blocks. - Schema prompt models repeated records as an array of objects. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hskm8VdJhy1krbeJZMXCcS --- .../classes/DocumentInterpreter.ts | 391 ++++++++++++------ 1 file changed, 276 insertions(+), 115 deletions(-) diff --git a/server/src/workflow-management/classes/DocumentInterpreter.ts b/server/src/workflow-management/classes/DocumentInterpreter.ts index c72ddde4..2d0574a6 100644 --- a/server/src/workflow-management/classes/DocumentInterpreter.ts +++ b/server/src/workflow-management/classes/DocumentInterpreter.ts @@ -28,6 +28,10 @@ const TARGET_OUTPUT_CHARS = 9000; const MIN_CHUNK_CHARS = 2500; const MAX_FILE_SIZE_MB = 10; const MAX_TABLES_IN_CONTEXT = 5; +const MAX_TABLE_ROWS_IN_CONTEXT = 50; +const MAX_TABLE_CONTEXT_CHARS = 8000; +const CHUNK_HEADER_ALLOWANCE = 64; +const MAX_PAGE_SUBCHUNKS = 20; const SCANNED_THRESHOLD_CHARS_PER_PAGE = 30; const MAX_TABLE_DETECTION_PAGES = 200; const EXTRACTION_CONCURRENCY = 3; @@ -125,7 +129,12 @@ const renderMarkdownTable = (rows: string[][]): string => { }; const escapeHtml = (value: string): string => - String(value ?? '').replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + String(value ?? '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); const renderHtmlTable = (rows: string[][]): string => { const [head, ...body] = rows; @@ -810,7 +819,8 @@ const normalizeValueBySchema = (value: any, field: ExtractionSchemaField): any = return typeof value === 'string' ? cleanText(value) : String(value); } }; - /** + +/** * Models often wrap the answer ({"data": {...}}, {"result": {...}}, * {"extracted": {...}}). Unwrap single-key envelopes until the schema keys * show up. @@ -884,7 +894,7 @@ const mergeValues = (current: any, incoming: any, field: ExtractionSchemaField): if (!isMeaningful(incoming)) return current; if (field.type === 'array') { - const base = Array.isArray(current) ? current : []; + const base = Array.isArray(current) ? current : []; const additions = Array.isArray(incoming) ? incoming : []; const collected = new Map(); base.forEach((item) => { @@ -946,6 +956,38 @@ const fallbackExtract = (text: string, schema: ExtractionSchema): Record { + if (text.length <= limit) return [text]; + + const segments: string[] = []; + let remaining = text; + const minimumCut = Math.floor(limit / 2); + + while (remaining.length > limit && segments.length < MAX_PAGE_SUBCHUNKS - 1) { + const candidates = [ + remaining.lastIndexOf('\n\n', limit), + remaining.lastIndexOf('\n', limit), + remaining.lastIndexOf(' ', limit), + ]; + const cut = candidates.find((index) => index >= minimumCut) ?? limit; + + segments.push(remaining.slice(0, cut).trim()); + remaining = remaining.slice(cut).trim(); + } + + if (remaining) segments.push(remaining); + return segments.filter(Boolean); +}; + const TABLE_ROW = /^\|.*\|\s*$/; const TABLE_SEPARATOR = /^\|(\s*:?-{3,}:?\s*\|)+\s*$/; @@ -970,34 +1012,55 @@ const carryTableHeaders = (segments: string[]): string[] => { const buildChunks = (pages: ParsedPage[], budget: number = MAX_CHUNK_CHARS): Array<{ pageRange: string; text: string }> => { const chunks: Array<{ pageRange: string; text: string }> = []; - let currentPages: ParsedPage[] = []; + const segmentBudget = budget - CHUNK_HEADER_ALLOWANCE; + + let currentUnits: Array<{ label: string; text: string }> = []; let currentLength = 0; + const flushCurrentUnits = () => { + if (currentUnits.length === 0) return; + + const firstLabel = currentUnits[0].label; + const lastLabel = currentUnits[currentUnits.length - 1].label; + + chunks.push({ + pageRange: firstLabel === lastLabel ? firstLabel : `${firstLabel} to ${lastLabel}`, + text: currentUnits.map((unit) => `${unit.label}\n${unit.text}`).join('\n\n'), + }); + + currentUnits = []; + currentLength = 0; + }; + for (const page of pages) { const pageText = cleanText(page.text); if (!pageText) continue; + const pageName = page.label ? `Sheet "${page.label}"` : `Page ${page.pageNumber}`; - const pageLength = pageText.length; - if (currentPages.length > 0 && currentLength + pageLength > MAX_CHUNK_CHARS) { - chunks.push({ - pageRange: `${currentPages[0].pageNumber} to ${currentPages[currentPages.length - 1].pageNumber}`, - text: currentPages.map((p) => `Page ${p.pageNumber}\n${cleanText(p.text)}`).join('\n\n'), - }); - currentPages = []; - currentLength = 0; + const segments = carryTableHeaders(splitOversizedPageText(pageText, segmentBudget)); + + if (segments.length >= MAX_PAGE_SUBCHUNKS) { + logger.warn( + `[DocumentInterpreter] Page ${page.pageNumber} exceeds ${MAX_PAGE_SUBCHUNKS} extraction chunks; content beyond that point is not sent to the model` + ); } - currentPages.push(page); - currentLength += pageLength; - } + segments.forEach((segment, index) => { + const label = segments.length > 1 + ? `${pageName} (part ${index + 1}/${segments.length})` + : pageName; - if (currentPages.length > 0) { - chunks.push({ - pageRange: `${currentPages[0].pageNumber}-${currentPages[currentPages.length - 1].pageNumber}`, - text: currentPages.map((p) => `Page ${p.pageNumber}\n${cleanText(p.text)}`).join('\n\n'), + if (currentUnits.length > 0 && currentLength + segment.length > segmentBudget) { + flushCurrentUnits(); + } + + currentUnits.push({ label, text: segment }); + currentLength += segment.length; }); } + flushCurrentUnits(); + return chunks.length > 0 ? chunks : [{ pageRange: '1-1', text: '' }]; }; @@ -1327,9 +1390,11 @@ class DocumentLLMClient { } const data = await response.json(); + const parsed = parseModelJson(data.message?.content || '', data.done_reason === 'length'); return { - payload: parseJsonLoose(data.message?.content || ''), + payload: parsed.payload, provider: `ollama:${model}`, + truncated: parsed.truncated, }; } finally { clearTimeout(timeoutId); @@ -1386,9 +1451,14 @@ class DocumentLLMClient { } const data = await response.json(); + const parsed = parseModelJson( + data.choices?.[0]?.message?.content || '', + data.choices?.[0]?.finish_reason === 'length' + ); return { - payload: parseJsonLoose(data.choices?.[0]?.message?.content || ''), + payload: parsed.payload, provider: `openai:${model}`, + truncated: parsed.truncated, }; } @@ -1411,9 +1481,11 @@ class DocumentLLMClient { }); const content = (response.content?.[0] as any)?.text || ''; + const parsed = parseModelJson(content, response.stop_reason === 'max_tokens'); return { - payload: parseJsonLoose(content), + payload: parsed.payload, provider: `anthropic:${model}`, + truncated: parsed.truncated, }; } @@ -1660,20 +1732,34 @@ export class DocumentInterpreter { } } - const pageTexts = textResult.pages.map((page) => ({ - pageNumber: page.num, - text: cleanText(page.text), - })); - - let tables: string[][][] = []; + // Tables per page. This used to read `mergedTables`, which is only + // filled for tables merged across pages and is empty otherwise, and only + // looked at the first 5 pages, so ordinary tables never reached the + // model or the Markdown. + const tablesByPage = new Map(); try { - const tableResult = await parser.getTable({ first: Math.min(pageCount, MAX_TABLES_IN_CONTEXT) }); - tables = tableResult.mergedTables.slice(0, MAX_TABLES_IN_CONTEXT); + const tableResult = await parser.getTable( + pageCount > MAX_TABLE_DETECTION_PAGES ? { first: MAX_TABLE_DETECTION_PAGES } : undefined + ); + for (const page of tableResult.pages || []) { + const tables = (page.tables || []).filter((table) => table.length > 0); + if (tables.length > 0) tablesByPage.set(page.num, tables); + } } catch (error: any) { logger.warn(`[DocumentInterpreter] Table extraction skipped: ${error.message}`); } - return { text: combinedText, pageCount, pages: pageTexts, tables }; + const pages: ParsedPage[] = textResult.pages.map((page) => { + const blocks = buildPdfPageBlocks(page.text, tablesByPage.get(page.num) || []); + return { pageNumber: page.num, text: renderBlocksMarkdown(blocks), blocks }; + }); + + return { + text: pages.map((page) => page.text).filter(Boolean).join('\n\n'), + pageCount, + pages, + tables: [], + }; } finally { await parser.destroy().catch(() => undefined); } @@ -1747,26 +1833,18 @@ export class DocumentInterpreter { tables.push(normalizedRows); - const sheetText = normalizedRows.map((row) => row.join('\t')).join('\n'); - if (sheetText) { - pages.push({ pageNumber: pages.length + 1, text: cleanText(sheetText) }); + if (normalizedRows.length > 0) { + const blocks: ContentBlock[] = [{ type: 'table', rows: normalizedRows }]; + pages.push({ pageNumber: pages.length + 1, label: sheetName, text: renderBlocksMarkdown(blocks), blocks }); } } - const text = tables - .map((table) => - table - .map((row) => row.join('\t')) - .join('\n') - ) - .filter(Boolean) - .join('\n\n'); - return { - text: cleanText(text), + text: pages.map((page) => `Sheet "${page.label}"\n${page.text}`).join('\n\n'), pageCount: pages.length || workbook.SheetNames.length, pages, - tables, + tables: [], + isSpreadsheet: true, }; } @@ -1834,14 +1912,15 @@ export class DocumentInterpreter { }; } - const sheetText = normalizedRows.map((row: string[]) => row.join('\t')).join('\n'); - const text = cleanText(sheetText); + const blocks: ContentBlock[] = [{ type: 'table', rows: normalizedRows }]; + const text = renderBlocksMarkdown(blocks); return { text, pageCount: 1, - pages: [{ pageNumber: 1, text }], - tables: [normalizedRows], + pages: [{ pageNumber: 1, text, blocks }], + tables: [], + isSpreadsheet: true, }; } @@ -1851,13 +1930,27 @@ export class DocumentInterpreter { mammoth.convertToHtml({ buffer }), ]); - const text = cleanText(rawTextResult.value || ''); + const rawText = cleanText(rawTextResult.value || ''); + const sourceHtml = (htmlResult.value || '').trim(); + + // Raw text flattens tables to one cell per line; Markdown keeps rows and + // columns, headings and lists for the model and the schema sample. + let text = rawText; + if (sourceHtml) { + try { + const markdown = promoteEmptyTableHeaders(await parseMarkdown(sourceHtml)).trim(); + if (markdown.length >= rawText.length * 0.6) text = markdown; + } catch (error: any) { + logger.warn(`[DocumentInterpreter] DOCX Markdown conversion failed, using raw text: ${error.message}`); + } + } + return { text, pageCount: 1, pages: [{ pageNumber: 1, text }], tables: [], - sourceHtml: (htmlResult.value || '').trim(), + sourceHtml, }; } @@ -1949,6 +2042,8 @@ export class DocumentInterpreter { '{"schema":{"field_key":{"label":"Field Label","type":"string|number|boolean|date|array|object","description":"...","required":true,"items":{"label":"...","type":"string"},"fields":{"nested_key":{"label":"...","type":"string"}}}}}', 'Use concise snake_case keys.', 'Prefer simple flat schemas unless the prompt clearly implies nested objects or arrays.', + 'When the goal covers repeated records (line items, transactions, rows of a table, a list of people or products), model them as one array field whose items are objects with one field per column or attribute; keep header-level facts (numbers, dates, parties, totals) as separate top-level fields.', + 'Tables in the sample are Markdown tables; use their column headers to name item fields.', ].join('\n'); const userPrompt = [ @@ -1992,20 +2087,32 @@ export class DocumentInterpreter { private static buildExtractionPrompt( prompt: string, schema: ExtractionSchema, - chunk: { pageRange: string; text: string }, + chunk: DocumentChunk, + partIndex: number, + partCount: number, tables: string[][][] ): { systemPrompt: string; userPrompt: string } { const systemPrompt = [ - 'You extract structured JSON from business documents.', - 'Return only valid JSON.', - 'Use the provided schema exactly for keys.', - 'If a value is missing, return null.', - 'For arrays, return arrays.', - 'Do not invent values not grounded in the provided text.', + 'You extract structured JSON from one part of a business document.', + 'Return only a JSON object whose keys are exactly the schema keys.', + 'Copy values as they appear in the text (keep currency, separators and date formats as written); do not calculate, convert or summarize.', + 'For array fields, return every matching item that appears in this part, in document order. Never sample, summarize or stop early. In a Markdown table, each body row is one item; the header row is not an item.', + 'If a value does not appear in this part, return null (an empty array for arrays). Do not invent values or carry them over from other parts.', ].join('\n'); - const tableContext = tables.length > 0 - ? `\nDetected tables:\n${tables.map((table, index) => `Table ${index + 1}\n${table.map((row) => row.join(' | ')).join('\n')}`).join('\n\n')}` + const boundedTables = tables + .slice(0, MAX_TABLES_IN_CONTEXT) + .map((table: string[][], index: number) => { + const rows = table.slice(0, MAX_TABLE_ROWS_IN_CONTEXT); + const omitted = table.length > MAX_TABLE_ROWS_IN_CONTEXT + ? `\n... (${table.length - MAX_TABLE_ROWS_IN_CONTEXT} more rows omitted)` + : ''; + return `Table ${index + 1}\n${renderMarkdownTable(rows)}${omitted}`; + }) + .join('\n\n'); + + const tableContext = boundedTables + ? truncate(`\nDetected tables:\n${boundedTables}`, MAX_TABLE_CONTEXT_CHARS) : ''; const userPrompt = [ @@ -2013,7 +2120,11 @@ export class DocumentInterpreter { '', `Schema: ${JSON.stringify(schemaToJsonDefinition(schema), null, 2)}`, '', - `Document chunk pages: ${chunk.pageRange}`, + `Return JSON shaped like: ${JSON.stringify(schemaSkeleton(schema))}`, + '', + partCount > 1 + ? `Document part ${partIndex + 1} of ${partCount} (${chunk.pageRange}):` + : `Document (${chunk.pageRange}):`, truncate(chunk.text, MAX_CHUNK_CHARS), tableContext, ].join('\n'); @@ -2091,29 +2202,30 @@ export class DocumentInterpreter { ): Promise { const parsedDocument = await this.parseDocument(buffer, documentMimeType); const schema = sanitizeSchema(extractionSchema, prompt); - const chunks = buildChunks(parsedDocument.pages); + const collectsRecords = Object.values(schema).some( + (field) => field.type === 'array' && field.items?.type === 'object' + ); + const chunks = buildChunks(parsedDocument.pages, collectsRecords ? recordChunkBudget(parsedDocument.pages, schema) : MAX_CHUNK_CHARS) + .filter((chunk) => chunk.text.trim()); + + // Tables now sit inside the page text where they belong. Only OCR'd + // single-page documents (images) still carry separate key/value tables; + // every chunk used to get the first 5 tables of the whole document, so + // models re-extracted page 1's rows in every chunk. + const contextTables = chunks.length <= 1 ? parsedDocument.tables : []; + if (chunks.length === 0 && contextTables.length > 0) { + chunks.push({ pageRange: '1', text: '' }); + } + const providers: string[] = []; + + // Chunks run in parallel (they were sequential) and merge in page order. + const results = await mapWithConcurrency(chunks, EXTRACTION_CONCURRENCY, (chunk, index) => + this.extractChunk(prompt, schema, chunk, index, chunks.length, contextTables, 0, providers, llmConfig) + ); let mergedData: Record = {}; - let providerLabel = 'heuristic-fallback'; - - for (const chunk of chunks) { - if (!chunk.text.trim() && parsedDocument.tables.length === 0) continue; - - const { systemPrompt, userPrompt } = this.buildExtractionPrompt( - prompt, - schema, - chunk, - parsedDocument.tables - ); - - try { - const result = await DocumentLLMClient.callStructuredJson(systemPrompt, userPrompt, llmConfig); - const normalized = normalizeResultToSchema(result.payload, schema); - mergedData = mergeExtractionResults(mergedData, normalized, schema); - providerLabel = result.provider; - } catch (error: any) { - logger.warn(`[DocumentInterpreter] Chunk extraction failed for pages ${chunk.pageRange}: ${error.message}`); - } + for (const result of results) { + if (result) mergedData = mergeExtractionResults(mergedData, result, schema); } if (!Object.values(mergedData).some(isMeaningful)) { @@ -2123,35 +2235,90 @@ export class DocumentInterpreter { return { data: normalizeExtractionData(mergedData), pageCount: parsedDocument.pageCount, - model: providerLabel, + model: providers[providers.length - 1] || 'heuristic-fallback', extractedAt: new Date().toISOString(), tabName: 'Document Data', }; } + /** + * Extracts one chunk. When the model's answer was cut off by its output + * limit (long tables), the chunk is re-extracted as two halves, up to + * MAX_TRUNCATION_SPLITS levels deep. Previously the cut-off JSON failed to + * parse and every value in that chunk was lost. + */ + private static async extractChunk( + prompt: string, + schema: ExtractionSchema, + chunk: DocumentChunk, + partIndex: number, + partCount: number, + contextTables: string[][][], + depth: number, + providers: string[], + llmConfig?: LLMConfig + ): Promise | null> { + const { systemPrompt, userPrompt } = this.buildExtractionPrompt(prompt, schema, chunk, partIndex, partCount, contextTables); + + let result: ProviderResult; + try { + result = await DocumentLLMClient.callStructuredJson(systemPrompt, userPrompt, llmConfig); + } catch (error: any) { + logger.warn(`[DocumentInterpreter] Chunk extraction failed for ${chunk.pageRange}: ${error.message}`); + return null; + } + providers.push(result.provider); + + const normalized = normalizeResultToSchema(unwrapPayload(result.payload, schema), schema); + + if (result.truncated && depth < MAX_TRUNCATION_SPLITS) { + const halves = splitChunkInTwo(chunk); + if (halves) { + logger.info(`[DocumentInterpreter] Output cut off for ${chunk.pageRange}; re-extracting in two halves`); + const parts = await Promise.all( + halves.map((half) => + this.extractChunk(prompt, schema, half, partIndex, partCount, contextTables, depth + 1, providers, llmConfig) + ) + ); + let merged: Record = {}; + for (const part of parts) { + if (part) merged = mergeExtractionResults(merged, part, schema); + } + // Keep scalars from the cut-off answer where the halves found none. + return mergeExtractionResults(merged, scalarsOnly(normalized, schema), schema); + } + logger.warn(`[DocumentInterpreter] Output cut off for ${chunk.pageRange}; keeping the complete part`); + } + + return normalized; + } + + /** + * Markdown for parse robots. PDF tables appear where they sit on the page + * (they were missing entirely before); spreadsheets render as one table per + * sheet instead of a tab-separated dump followed by the same data again as + * a table. + */ private static async toMarkdown(doc: ParsedDocument): Promise { if (doc.sourceHtml) { - return parseMarkdown(doc.sourceHtml); + return promoteEmptyTableHeaders(await parseMarkdown(doc.sourceHtml)); } const parts: string[] = []; for (const page of doc.pages) { - const text = cleanText(page.text); - if (!text) continue; - parts.push(doc.isImage ? text : `## Page ${page.pageNumber}\n\n${text}`); + const body = page.blocks ? renderBlocksMarkdown(page.blocks) : tidyLines(page.text); + if (!body) continue; + if (doc.isSpreadsheet) { + parts.push(page.label && doc.pages.length > 1 ? `## ${page.label}\n\n${body}` : body); + } else { + parts.push(doc.isImage ? body : `## Page ${page.pageNumber}\n\n${body}`); + } } - for (let i = 0; i < doc.tables.length; i++) { - const table = doc.tables[i]; - if (table.length === 0) continue; - const header = table[0].map((cell) => cell.trim() || ' ').join(' | '); - const separator = table[0].map(() => '---').join(' | '); - const rows = table.slice(1).map((row) => - row.map((cell) => cell.trim().replace(/\n/g, ' ') || ' ').join(' | ') - ); - parts.push(`### Table ${i + 1}\n\n| ${header} |\n| ${separator} |\n${rows.map((r) => `| ${r} |`).join('\n')}`); - } + doc.tables.forEach((table, index) => { + if (table.length > 0) parts.push(`### Table ${index + 1}\n\n${renderMarkdownTable(table)}`); + }); return parts.join('\n\n'); } @@ -2162,29 +2329,23 @@ export class DocumentInterpreter { const parts: string[] = ['
']; for (const page of doc.pages) { - const text = cleanText(page.text); - if (!text) continue; - const paragraphs = text - .split(/\n{2,}/) - .map((p) => `

${escapeHtml(p).replace(/\n/g, '
')}

`) - .join('\n'); + const blocks: ContentBlock[] = page.blocks || [{ type: 'text', text: tidyLines(page.text) }]; + const body = renderBlocksHtml(blocks); + if (!body.trim()) continue; + if (doc.isSpreadsheet) { + parts.push(page.label && doc.pages.length > 1 ? `
\n

${escapeHtml(page.label)}

\n${body}\n
` : body); + } else { parts.push( doc.isImage - ? paragraphs - : `
\n

Page ${page.pageNumber}

\n${paragraphs}\n
` + ? body + : `
\n

Page ${page.pageNumber}

\n${body}\n
` ); + } } - for (let i = 0; i < doc.tables.length; i++) { - const table = doc.tables[i]; - if (table.length === 0) continue; - const header = `${table[0].map((cell) => `${escapeHtml(cell.trim())}`).join('')}`; - const body = `${table - .slice(1) - .map((row) => `${row.map((cell) => `${escapeHtml(cell.trim())}`).join('')}`) - .join('\n')}`; - parts.push(`\n${header}\n${body}\n
`); - } + doc.tables.forEach((table) => { + if (table.length > 0) parts.push(renderHtmlTable(table)); + }); parts.push('
'); return parts.join('\n'); From 8bca23e99c83d188713003399e50eb7ea883f776 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 08:53:17 +0000 Subject: [PATCH 3/4] fix(documents): address review on chunk sizing, retries and JSON salvage - buildChunks counts each unit's label and separator against the budget; the prompt's chunk cap is now a safety net above it, so packed chunks no longer lose their last rows. - Retry halves of a cut-off chunk run one at a time, keeping requests within EXTRACTION_CONCURRENCY. - salvageTruncatedJson cuts only between array elements or top-level fields, so a record cut mid-way is dropped instead of kept with missing fields. - extractBalancedJson skips brackets in prose ("[see notes]") and only treats an unclosed JSON-looking value as cut-off output. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hskm8VdJhy1krbeJZMXCcS --- .../classes/DocumentInterpreter.ts | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/server/src/workflow-management/classes/DocumentInterpreter.ts b/server/src/workflow-management/classes/DocumentInterpreter.ts index 2d0574a6..090de8df 100644 --- a/server/src/workflow-management/classes/DocumentInterpreter.ts +++ b/server/src/workflow-management/classes/DocumentInterpreter.ts @@ -31,6 +31,7 @@ const MAX_TABLES_IN_CONTEXT = 5; const MAX_TABLE_ROWS_IN_CONTEXT = 50; const MAX_TABLE_CONTEXT_CHARS = 8000; const CHUNK_HEADER_ALLOWANCE = 64; +const MAX_PROMPT_CHUNK_HEADROOM = 2000; const MAX_PAGE_SUBCHUNKS = 20; const SCANNED_THRESHOLD_CHARS_PER_PAGE = 30; const MAX_TABLE_DETECTION_PAGES = 200; @@ -463,6 +464,8 @@ const extractBalancedJson = (value: string): string | null => { for (let start = 0; start < value.length; start += 1) { const opener = value[start]; if (opener !== '{' && opener !== '[') continue; + // "[see below]"-style brackets in prose aren't JSON; keep scanning. + if (!/^\s*(?:["{\[\]}\d-]|$)/.test(value.slice(start + 1, start + 20))) continue; const closer = closerFor[opener]; let depth = 0; @@ -486,7 +489,7 @@ const extractBalancedJson = (value: string): string | null => { if (depth === 0) return value.slice(start, i + 1); } } - // The outermost value never closed: the output was cut off. Don't fall + // The outermost JSON value never closed: the output was cut off. Don't fall // through to the first complete *inner* object, which silently turned a // truncated 100-item list into a single item. return null; @@ -500,6 +503,9 @@ const extractBalancedJson = (value: string): string | null => { * `{"items":[{"a":1},{"a":2` → `{"items":[{"a":1}]}`. */ const salvageTruncatedJson = (value: string): any | null => { + // Cut only between array elements or top-level fields, so a record that + // was still being written is dropped rather than kept with missing fields. + const isElementBoundary = (stack: string[]) => stack.length === 1 || stack[stack.length - 1] === ']'; const start = value.search(/[{[]/); if (start < 0) return null; const text = value.slice(start); @@ -523,9 +529,11 @@ const salvageTruncatedJson = (value: string): any | null => { else if (char === '}' || char === ']') { stack.pop(); if (stack.length === 0) return null; // complete, not truncated - lastCut = i + 1; - stackAtCut = [...stack]; - } else if (char === ',') { + if (isElementBoundary(stack)) { + lastCut = i + 1; + stackAtCut = [...stack]; + } + } else if (char === ',' && isElementBoundary(stack)) { lastCut = i; stackAtCut = [...stack]; } @@ -1050,12 +1058,14 @@ const buildChunks = (pages: ParsedPage[], budget: number = MAX_CHUNK_CHARS): Arr ? `${pageName} (part ${index + 1}/${segments.length})` : pageName; - if (currentUnits.length > 0 && currentLength + segment.length > segmentBudget) { + // Count the label and separator each unit adds to the chunk text. + const unitLength = label.length + 1 + segment.length + 2; + if (currentUnits.length > 0 && currentLength + unitLength > budget) { flushCurrentUnits(); } currentUnits.push({ label, text: segment }); - currentLength += segment.length; + currentLength += unitLength; }); } @@ -2125,7 +2135,9 @@ export class DocumentInterpreter { partCount > 1 ? `Document part ${partIndex + 1} of ${partCount} (${chunk.pageRange}):` : `Document (${chunk.pageRange}):`, - truncate(chunk.text, MAX_CHUNK_CHARS), + // Safety cap only: chunks are packed to the budget, but a carried table + // header can push one past it, and cutting here would drop its last rows. + truncate(chunk.text, MAX_CHUNK_CHARS + MAX_PROMPT_CHUNK_HEADROOM), tableContext, ].join('\n'); @@ -2275,11 +2287,13 @@ export class DocumentInterpreter { const halves = splitChunkInTwo(chunk); if (halves) { logger.info(`[DocumentInterpreter] Output cut off for ${chunk.pageRange}; re-extracting in two halves`); - const parts = await Promise.all( - halves.map((half) => - this.extractChunk(prompt, schema, half, partIndex, partCount, contextTables, depth + 1, providers, llmConfig) - ) - ); + // One half at a time, so retries stay within EXTRACTION_CONCURRENCY. + const parts: Array | null> = []; + for (const half of halves) { + parts.push( + await this.extractChunk(prompt, schema, half, partIndex, partCount, contextTables, depth + 1, providers, llmConfig) + ); + } let merged: Record = {}; for (const part of parts) { if (part) merged = mergeExtractionResults(merged, part, schema); From d77785004498dca712a55d915c21353847b7dffe Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 09:06:43 +0000 Subject: [PATCH 4/4] fix(documents): split oversized pages fully instead of capping at 20 segments A page over 20 segments put the rest of the page in the last one, which the prompt then cut, so a large CSV or sheet silently lost its tail (a 6,000-row CSV came back with 2,302 rows). Pages are now split into as many segments as they need, as multi-page documents already are. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hskm8VdJhy1krbeJZMXCcS --- .../classes/DocumentInterpreter.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/server/src/workflow-management/classes/DocumentInterpreter.ts b/server/src/workflow-management/classes/DocumentInterpreter.ts index 090de8df..c06df508 100644 --- a/server/src/workflow-management/classes/DocumentInterpreter.ts +++ b/server/src/workflow-management/classes/DocumentInterpreter.ts @@ -32,7 +32,6 @@ const MAX_TABLE_ROWS_IN_CONTEXT = 50; const MAX_TABLE_CONTEXT_CHARS = 8000; const CHUNK_HEADER_ALLOWANCE = 64; const MAX_PROMPT_CHUNK_HEADROOM = 2000; -const MAX_PAGE_SUBCHUNKS = 20; const SCANNED_THRESHOLD_CHARS_PER_PAGE = 30; const MAX_TABLE_DETECTION_PAGES = 200; const EXTRACTION_CONCURRENCY = 3; @@ -972,6 +971,8 @@ const fallbackExtract = (text: string, schema: ExtractionSchema): Record { if (text.length <= limit) return [text]; @@ -980,7 +981,7 @@ const splitOversizedPageText = (text: string, limit: number): string[] => { let remaining = text; const minimumCut = Math.floor(limit / 2); - while (remaining.length > limit && segments.length < MAX_PAGE_SUBCHUNKS - 1) { + while (remaining.length > limit) { const candidates = [ remaining.lastIndexOf('\n\n', limit), remaining.lastIndexOf('\n', limit), @@ -1047,12 +1048,6 @@ const buildChunks = (pages: ParsedPage[], budget: number = MAX_CHUNK_CHARS): Arr const segments = carryTableHeaders(splitOversizedPageText(pageText, segmentBudget)); - if (segments.length >= MAX_PAGE_SUBCHUNKS) { - logger.warn( - `[DocumentInterpreter] Page ${page.pageNumber} exceeds ${MAX_PAGE_SUBCHUNKS} extraction chunks; content beyond that point is not sent to the model` - ); - } - segments.forEach((segment, index) => { const label = segments.length > 1 ? `${pageName} (part ${index + 1}/${segments.length})`