From 463e12044e3842b1a7a0d251bc10a4a2bbd3d99a Mon Sep 17 00:00:00 2001 From: Francisco Solla Date: Tue, 29 Sep 2026 22:26:48 -0300 Subject: [PATCH] =?UTF-8?q?feat(C237):=20vetor=20da=20fala=20inclui=20o=20?= =?UTF-8?q?sum=C3=A1rio/keywords=20oficiais?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Fase 5 pós-deploy do C229 mediu o corpus real e o aceite 'impeachment' falhava: a 222 (votação do impeachment de Dilma, 17/04/2016) ficava em #841 porque o índice só embutia o ASR — e a fala nunca pronuncia 'impeachment'; a relevância dela mora no sumário/keywords oficiais da Câmara. O plano do índice passa a carregar metadataText (sumário — ou título no web — + keywords, incluído no hash de idempotência) e o vetor da fala vira normalize(média das unidades + 2·metadata), calibrado no corpus real (w=2: 222 em #8; w=1: #25). A metadata é vetorizada no mesmo lote do provedor mas não vira unidade armazenada nem evidência; sem migration. Related #1395 --- docs/changelog/2026-09-30-c237.md | 1 + scripts/index-speeches.mjs | 12 +++- scripts/lib/speechIndex.mjs | 12 +++- src/lib/speechSemantic.ts | 62 ++++++++++++++++++- src/utilities/speech/speechEmbeddingIndex.ts | 32 ++++++++-- tests/int/speechAcervo.int.spec.ts | 64 ++++++++++++++++++++ tests/unit/speechIndex.unit.spec.ts | 21 +++++++ tests/unit/speechSemantic.unit.spec.ts | 64 ++++++++++++++++++++ 8 files changed, 257 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/2026-09-30-c237.md diff --git a/docs/changelog/2026-09-30-c237.md b/docs/changelog/2026-09-30-c237.md new file mode 100644 index 00000000..40b6991a --- /dev/null +++ b/docs/changelog/2026-09-30-c237.md @@ -0,0 +1 @@ +**C237 — Acervo por tema: o vetor da fala passa a incluir o sumário/keywords oficiais (2026-09-30):** a Fase 5 pós-deploy do C229 mediu o corpus real e o aceite "impeachment → fala da votação de Dilma" falhava: a 222 (17/04/2016) ficava em #841 e a 216 (14/04/2016) em #82, enquanto 8 das 10 primeiras eram falas que sequer citam impeachment/impedimento (golpe, cassação, PGR/Temer) — o índice só embutia o ASR, e a 222 nunca pronuncia "impeachment"; a relevância dela mora no **sumário/keywords oficiais da Câmara**, que nunca entravam no vetor. Fix: `planSpeechIndex` ganha `metadataText` (sumário — ou título no web — + keywords cruas; `speechMetadataText`) e o **hash de idempotência inclui a metadata** (editar o sumário reindexa); o vetor da fala passa a ser `normalize(média_das_unidades + 2·metadata)` (`composeSpeechVector`, `SPEECH_METADATA_VECTOR_WEIGHT = 2` calibrado no corpus real: w=2 leva a 222 à #8 com o cluster do tema na frente; w=1 deixava em #25); a metadata é vetorizada no mesmo lote do provedor mas **não vira unidade armazenada nem evidência** (o "Trecho mais próximo do tema" continua falado) e não há migration. CLI `acervo:index` seleciona/passa `title`/`keywords` e o relatório ganha `(+N metadados)` ao lado dos trechos (dry-run e write), com testes unit do plano/hash/composição/relatório e int do ranking (fala cujo ASR não cita o tema vence pela metadata). Reindexar produção após o deploy (hash muda para toda fala com metadata; ~US$ 0,007 e ~31 min no corpus atual). Evidência da Fase 5 e aceite na Issue #1395. diff --git a/scripts/index-speeches.mjs b/scripts/index-speeches.mjs index f5a5812f..411ebbd6 100644 --- a/scripts/index-speeches.mjs +++ b/scripts/index-speeches.mjs @@ -80,7 +80,14 @@ const loadSources = async (payload, { source, limit }) => { depth: 0, ...(limit ? { limit, page: 1 } : { limit: 0, pagination: false }), sort: 'speechAt', - select: { sourceKey: true, officialTranscript: true, summary: true, searchText: true }, + select: { + sourceKey: true, + officialTranscript: true, + summary: true, + title: true, + keywords: true, + searchText: true, + }, // Intentional bypass: the index CLI is a trusted actor with no session. overrideAccess: true, }) @@ -116,6 +123,8 @@ const loadSources = async (payload, { source, limit }) => { sourceKey: speech.sourceKey, officialTranscript: speech.officialTranscript, summary: speech.summary, + title: speech.title, + keywords: speech.keywords, searchText: speech.searchText, segments: segmentsBySpeech.get(speech.id) ?? [], })) @@ -204,6 +213,7 @@ async function main() { indexed: result.indexed, failed: result.failed, units: result.units, + metadataTexts: result.metadataTexts, estimatedBytes: speechIndexEstimatedBytes(result.units, DEEPINFRA_EMBED_DIMENSIONS), promptTokens: result.promptTokens, costUsd: speechIndexCostUsd(result.promptTokens, DEEPINFRA_EMBED_COST_PER_MILLION_TOKENS_USD), diff --git a/scripts/lib/speechIndex.mjs b/scripts/lib/speechIndex.mjs index 10e04479..b1fa4fe4 100644 --- a/scripts/lib/speechIndex.mjs +++ b/scripts/lib/speechIndex.mjs @@ -71,11 +71,16 @@ export const speechIndexCostUsd = (promptTokens, pricePerMillionTokensUsd) => /** Rough JSON cost of the vectors: units × dims × ~8 bytes per number. */ export const speechIndexEstimatedBytes = (units, dimensions) => units * dimensions * 8 +/** C237 — the official metadata rides the same provider call but is not a stored unit. */ +const metadataSuffix = (metadataTexts) => + typeof metadataTexts === 'number' && metadataTexts > 0 ? ` (+${metadataTexts} metadados)` : '' + /** * @param {{ * target: string, mode: 'index' | 'dry-run', source: string, * speeches: number, upToDate: number, skippedNoText: number, toIndex: number, - * units: number, estimatedBytes?: number, indexed?: number, failed?: number, + * units: number, metadataTexts?: number, estimatedBytes?: number, + * indexed?: number, failed?: number, * promptTokens?: number, costUsd?: number, durationMs: number, * }} report * @returns {string[]} @@ -85,6 +90,7 @@ export const formatSpeechIndexReport = (report) => { typeof report.estimatedBytes === 'number' ? ` | bytes estimados: ${(report.estimatedBytes / 1_000_000).toFixed(1)} MB` : '' + const metadata = metadataSuffix(report.metadataTexts) const lines = [ `[acervo:index] alvo: ${report.target} | modo: ${report.mode} | fonte: ${report.source}`, `falas: ${report.speeches} | atualizadas: ${report.upToDate} | sem texto: ${report.skippedNoText} | a indexar: ${report.toIndex}`, @@ -92,10 +98,10 @@ export const formatSpeechIndexReport = (report) => { if (report.mode === 'index') { lines.push( - `indexadas: ${report.indexed ?? 0} | falhas: ${report.failed ?? 0} | trechos vetorizados: ${report.units}${bytes} | tokens: ${report.promptTokens ?? 0} | custo estimado: US$ ${(report.costUsd ?? 0).toFixed(4)}`, + `indexadas: ${report.indexed ?? 0} | falhas: ${report.failed ?? 0} | trechos vetorizados: ${report.units}${metadata}${bytes} | tokens: ${report.promptTokens ?? 0} | custo estimado: US$ ${(report.costUsd ?? 0).toFixed(4)}`, ) } else { - lines.push(`trechos a vetorizar: ${report.units}${bytes}`) + lines.push(`trechos a vetorizar: ${report.units}${metadata}${bytes}`) } lines.push(`duração: ${(report.durationMs / 1000).toFixed(1)}s`) diff --git a/src/lib/speechSemantic.ts b/src/lib/speechSemantic.ts index 4bede083..2579b40d 100644 --- a/src/lib/speechSemantic.ts +++ b/src/lib/speechSemantic.ts @@ -11,6 +11,16 @@ /** The provisional cut-off of the theme mode — calibrated by `acervo:index --probe`. */ export const SPEECH_SEMANTIC_MIN_COSINE = 0.4 +/** + * C237 — weight of the official metadata (summary/title + keywords) against the + * mean of the spoken units when composing the speech-level vector. Calibrated + * on the production corpus (2026-09-30): with weight 1 the Dilma impeachment + * vote speech (222) sat at #25 for the theme "impeachment"; with 2 it reaches + * the first page (#8) and the topic cluster (2016 process + Bolsonaro requests) + * leads, without dropping the other acceptance themes out of their cluster. + */ +export const SPEECH_METADATA_VECTOR_WEIGHT = 2 + /** Text windows of speeches without ASR segments: ~1200 chars, ~200 of overlap. */ const SPEECH_WINDOW_SIZE_CHARS = 1200 const SPEECH_WINDOW_OVERLAP_CHARS = 200 @@ -239,10 +249,50 @@ type SpeechIndexUnit = { export type SpeechIndexPlan = { units: SpeechIndexUnit[] - /** Hash of every unit text + model: the CLI's skip key. */ + /** + * C237 — official metadata text folded into the speech-level vector (never + * stored as a unit, never evidence). `null` when the speech carries none. + */ + metadataText: string | null + /** Hash of every unit text + metadata + model: the CLI's skip key. */ speechHash: string } +/** + * C237 — the official metadata of one speech: the Câmara summary (the web title + * when there is no summary) plus the raw official keywords. This is the text + * that states the *topic* even when the spoken passage never names it (the + * impeachment vote speech never says "impeachment"). + */ +export const speechMetadataText = (speech: { + summary?: string | null + title?: string | null + keywords?: readonly string[] | null +}): string | null => { + const head = [speech.summary, speech.title].find( + (candidate) => typeof candidate === 'string' && candidate.trim() !== '', + ) + const keywords = (speech.keywords ?? []).map((keyword) => keyword.trim()).filter(Boolean) + const text = [head?.trim() ?? '', keywords.join(', ')].filter(Boolean).join('\n') + return text === '' ? null : text +} + +/** + * C237 — the speech-level vector of the index: the normalized mean of the unit + * vectors plus the weighted official metadata. A speech without metadata keeps + * the pre-C237 mean exactly. + */ +export const composeSpeechVector = ( + unitVectors: readonly SpeechSemanticVector[], + metadataVector: SpeechSemanticVector | null, +): number[] => { + const mean = meanPoolVectors(unitVectors) + if (!metadataVector || metadataVector.length !== mean.length || mean.length === 0) return mean + return normalizeVector( + mean.map((value, index) => value + SPEECH_METADATA_VECTOR_WEIGHT * metadataVector[index]), + ) +} + export type SpeechIndexSegment = { order: number startSeconds: number @@ -259,6 +309,8 @@ export const planSpeechIndex = ( speech: { officialTranscript?: string | null summary?: string | null + title?: string | null + keywords?: readonly string[] | null searchText?: string | null segments?: readonly SpeechIndexSegment[] }, @@ -289,9 +341,13 @@ export const planSpeechIndex = ( if (units.length === 0) return null + const metadataText = speechMetadataText(speech) const speechHash = speechEmbeddingHash( - units.map((unit) => `${unit.kind}:${unit.order}:${unit.text}`).join('\n'), + [ + ...units.map((unit) => `${unit.kind}:${unit.order}:${unit.text}`), + ...(metadataText ? [`metadata:${metadataText}`] : []), + ].join('\n'), model, ) - return { units, speechHash } + return { units, metadataText, speechHash } } diff --git a/src/utilities/speech/speechEmbeddingIndex.ts b/src/utilities/speech/speechEmbeddingIndex.ts index 3f5a8903..4ccc3354 100644 --- a/src/utilities/speech/speechEmbeddingIndex.ts +++ b/src/utilities/speech/speechEmbeddingIndex.ts @@ -3,8 +3,8 @@ import 'server-only' import type { Payload } from 'payload' import { + composeSpeechVector, cosineSimilarity, - meanPoolVectors, planSpeechIndex, type SpeechIndexPlan, type SpeechIndexSegment, @@ -24,6 +24,10 @@ export type SpeechIndexSource = { id: number officialTranscript?: string | null summary?: string | null + /** C237 — web title, the metadata head when there is no summary. */ + title?: string | null + /** C237 — raw official keywords, folded into the metadata text. */ + keywords?: readonly string[] | null searchText?: string | null segments: readonly SpeechIndexSegment[] } @@ -41,6 +45,8 @@ export type SpeechIndexPlanBatch = { /** No usable text at all. */ skippedNoText: number units: number + /** C237 — one extra provider input per stale speech that has metadata. */ + metadataTexts: number } export type SpeechIndexBatchResult = { @@ -48,7 +54,10 @@ export type SpeechIndexBatchResult = { upToDate: number skippedNoText: number failed: number + /** Stored retrieval units (segments/windows). */ units: number + /** C237 — metadata texts embedded on top of the units (not stored). */ + metadataTexts: number promptTokens: number } @@ -120,6 +129,7 @@ export const planSpeechIndexBatch = async ( upToDate: planned.length - stale.length, skippedNoText, units: stale.reduce((total, { plan }) => total + plan.units.length, 0), + metadataTexts: stale.reduce((total, { plan }) => total + (plan.metadataText ? 1 : 0), 0), } } @@ -141,29 +151,41 @@ export const indexSpeechSources = async ( upToDate: batch.upToDate, skippedNoText: batch.skippedNoText, units: 0, + metadataTexts: 0, promptTokens: 0, } if (batch.stale.length === 0) return { ...base, indexed: 0, failed: 0 } - const texts = batch.stale.flatMap(({ plan }) => plan.units.map((unit) => unit.text)) + // C237 — the official metadata rides the same ordered provider call, right + // after the units of each speech; it feeds the speech vector only. + const texts = batch.stale.flatMap(({ plan }) => [ + ...plan.units.map((unit) => unit.text), + ...(plan.metadataText ? [plan.metadataText] : []), + ]) const embedded = await embedTexts(texts) if (!embedded) { return { ...base, indexed: 0, failed: batch.stale.length } } // The whole batch was embedded already: the tokens were spent even if a // later write fails. - const spent = { units: texts.length, promptTokens: embedded.promptTokens } + const spent = { + units: batch.units, + metadataTexts: batch.metadataTexts, + promptTokens: embedded.promptTokens, + } let cursor = 0 let indexed = 0 for (const { source, plan } of batch.stale) { const vectors = embedded.vectors.slice(cursor, cursor + plan.units.length) cursor += plan.units.length + const metadataVector = plan.metadataText ? (embedded.vectors[cursor] ?? null) : null + if (plan.metadataText) cursor += 1 if (vectors.length !== plan.units.length) { return { ...base, ...spent, indexed, failed: batch.stale.length - indexed } } - const speechVector = meanPoolVectors(vectors) + const speechVector = composeSpeechVector(vectors, metadataVector) await withPayloadTransaction(payload, async ({ req }) => { await payload.delete({ collection: 'speechEmbedding', @@ -299,6 +321,7 @@ export type SpeechIndexPreview = { skippedNoText: number toIndex: number units: number + metadataTexts: number } export const previewSpeechIndex = ( @@ -310,4 +333,5 @@ export const previewSpeechIndex = ( skippedNoText: batch.skippedNoText, toIndex: batch.stale.length, units: batch.units, + metadataTexts: batch.metadataTexts, }) diff --git a/tests/int/speechAcervo.int.spec.ts b/tests/int/speechAcervo.int.spec.ts index de8a5b66..f348da7f 100644 --- a/tests/int/speechAcervo.int.spec.ts +++ b/tests/int/speechAcervo.int.spec.ts @@ -613,6 +613,70 @@ describe('speech acervo (C154)', () => { ).toContain(`Primeiro trecho ${runId}`) }) + it('folds the official metadata into the speech vector (C237)', async () => { + const runId = randomUUID().slice(0, 8) + const offTopic = [0.5, Math.sqrt(3) / 2] + const summary = `Processo de impeachment da Presidenta Dilma Rousseff ${runId}` + const idWithMetadata = await createSpeech({ + speechAt: '1988-01-01T10:00', + year: 1988, + summary, + keywords: ['IMPEACHMENT'], + segments: [{ startSeconds: 0, endSeconds: 3, text: `Trecho sem o tema ${runId}` }], + }) + const idWithout = await createSpeech({ + speechAt: '1988-01-02T10:00', + year: 1988, + segments: [{ startSeconds: 0, endSeconds: 3, text: `Trecho sem o tema ${runId}` }], + }) + const embedCalls: string[][] = [] + const embedTexts = async (texts: readonly string[]) => { + embedCalls.push([...texts]) + return { + vectors: texts.map((text) => + text.includes(runId) && text.startsWith('Processo') ? [1, 0] : offTopic, + ), + promptTokens: 5, + } + } + + const result = await indexSpeechSources( + payload, + [ + { + id: idWithMetadata, + summary, + keywords: ['IMPEACHMENT'], + segments: [{ order: 1, startSeconds: 0, text: `Trecho sem o tema ${runId}` }], + }, + { + id: idWithout, + segments: [{ order: 1, startSeconds: 0, text: `Trecho sem o tema ${runId}` }], + }, + ], + { embedTexts }, + ) + + expect(result).toMatchObject({ indexed: 2, failed: 0, units: 2, metadataTexts: 1 }) + expect(embedCalls[0]).toEqual([ + `Trecho sem o tema ${runId}`, + `${summary}\nIMPEACHMENT`, + `Trecho sem o tema ${runId}`, + ]) + + const { coordinator } = await createUsers() + const data = await loadSpeechAcervoPageData( + payload, + coordinator, + { q: `tema ${runId}`, mode: 'tema', year: '1988' }, + async () => [1, 0], + ) + // Both passages never state the theme; the official metadata is what ranks + // the speech that carries it above the identical spoken-only neighbour. + expect(data.rows.map((row) => row.id)).toEqual([idWithMetadata, idWithout]) + expect(data.rows[0]?.semanticMatch).toBe(true) + }) + it('cascades the index rows when the speech is deleted (C229)', async () => { const id = await createSpeech() await seedSpeechEmbedding(id, 'speech', null, [1, 0]) diff --git a/tests/unit/speechIndex.unit.spec.ts b/tests/unit/speechIndex.unit.spec.ts index 7c2581ca..e2c0816b 100644 --- a/tests/unit/speechIndex.unit.spec.ts +++ b/tests/unit/speechIndex.unit.spec.ts @@ -112,6 +112,27 @@ describe('formatSpeechIndexReport', () => { expect(lines[2]).toContain('tokens: 123456') expect(lines[2]).toContain('custo estimado: US$ 0.0012') }) + + it('reports the metadata inputs on top of the stored units (C237)', () => { + const dryRun = formatSpeechIndexReport({ ...base, mode: 'dry-run', metadataTexts: 42 }) + expect(dryRun[2]).toContain('trechos a vetorizar: 300 (+42 metadados)') + + const write = formatSpeechIndexReport({ + ...base, + mode: 'index', + metadataTexts: 42, + indexed: 15, + failed: 0, + promptTokens: 10, + costUsd: 0, + }) + expect(write[2]).toContain('trechos vetorizados: 300 (+42 metadados)') + + // No metadata (speech without summary/title/keywords) keeps the old shape. + expect(formatSpeechIndexReport({ ...base, mode: 'dry-run' })[2]).toBe( + 'trechos a vetorizar: 300', + ) + }) }) describe('formatSpeechIndexProbe', () => { diff --git a/tests/unit/speechSemantic.unit.spec.ts b/tests/unit/speechSemantic.unit.spec.ts index b749eb91..f9489f5b 100644 --- a/tests/unit/speechSemantic.unit.spec.ts +++ b/tests/unit/speechSemantic.unit.spec.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest' import { + composeSpeechVector, cosineSimilarity, dotProduct, meanPoolVectors, @@ -11,8 +12,10 @@ import { planSpeechIndex, rankSemanticHits, sortSemanticHits, + SPEECH_METADATA_VECTOR_WEIGHT, SPEECH_SEMANTIC_MIN_COSINE, speechEmbeddingHash, + speechMetadataText, speechSemanticSourceText, speechTextWindows, type SpeechSemanticCandidate, @@ -246,4 +249,65 @@ describe('planSpeechIndex', () => { planSpeechIndex(speech, { model: 'b' })?.speechHash, ) }) + + it('plans the official metadata and pins it in the hash (C237)', () => { + const speech = { + summary: 'Processo de impeachment da Presidenta Dilma Rousseff.', + keywords: ['IMPEACHMENT', 'DILMA ROUSSEFF'], + segments: [{ order: 1, startSeconds: 0, text: 'Sr. Presidente...' }], + } + const plan = planSpeechIndex(speech, { model }) + + expect(plan?.metadataText).toBe( + 'Processo de impeachment da Presidenta Dilma Rousseff.\nIMPEACHMENT, DILMA ROUSSEFF', + ) + expect(plan?.speechHash).toBe( + speechEmbeddingHash( + 'segment:1:Sr. Presidente...\nmetadata:Processo de impeachment da Presidenta Dilma Rousseff.\nIMPEACHMENT, DILMA ROUSSEFF', + model, + ), + ) + // Editing the summary or the keywords must reindex the speech. + expect( + planSpeechIndex({ ...speech, summary: 'Outro sumário' }, { model })?.speechHash, + ).not.toBe(plan?.speechHash) + expect(planSpeechIndex({ ...speech, keywords: ['OUTRA'] }, { model })?.speechHash).not.toBe( + plan?.speechHash, + ) + // A speech without metadata keeps the pre-C237 hash bytes. + expect(planSpeechIndex({ segments: speech.segments }, { model })?.metadataText).toBeNull() + expect(planSpeechIndex({ segments: speech.segments }, { model })?.speechHash).toBe( + speechEmbeddingHash('segment:1:Sr. Presidente...', model), + ) + }) +}) + +describe('speechMetadataText (C237)', () => { + it('prefers the summary over the title and joins the raw keywords', () => { + expect( + speechMetadataText({ summary: 'Resumo', title: 'Título', keywords: ['SUS', ' Saúde '] }), + ).toBe('Resumo\nSUS, Saúde') + expect(speechMetadataText({ title: 'Título', keywords: [] })).toBe('Título') + expect(speechMetadataText({ summary: ' ', keywords: [' '] })).toBeNull() + expect(speechMetadataText({})).toBeNull() + }) +}) + +describe('composeSpeechVector (C237)', () => { + const unit = [0.6, 0.8] + + it('keeps the pure mean without metadata', () => { + expect(composeSpeechVector([unit], null)).toEqual(meanPoolVectors([unit])) + // A metadata vector of another dimension never poisons the mean. + expect(composeSpeechVector([unit], [1, 0, 0])).toEqual(meanPoolVectors([unit])) + }) + + it('pulls the speech vector toward the weighted metadata', () => { + const composed = composeSpeechVector([unit], [1, 0]) + + expect(composed).toEqual( + normalizeVector([unit[0] + SPEECH_METADATA_VECTOR_WEIGHT * 1, unit[1]]), + ) + expect(cosineSimilarity(composed, [1, 0])).toBeGreaterThan(cosineSimilarity(unit, [1, 0])) + }) })