feat(C237): vetor da fala inclui o sumário/keywords oficiais

A Fase 5 pós-deploy do C229 mediu o corpus real e o aceite 'impeachment'
falhava: a 222 (votação do impeachment de Dilma, 17/04/2016) ficava em #841
porque o índice só embutia o ASR — e a fala nunca pronuncia 'impeachment';
a relevância dela mora no sumário/keywords oficiais da Câmara.

O plano do índice passa a carregar metadataText (sumário — ou título no web
— + keywords, incluído no hash de idempotência) e o vetor da fala vira
normalize(média das unidades + 2·metadata), calibrado no corpus real
(w=2: 222 em #8; w=1: #25). A metadata é vetorizada no mesmo lote do
provedor mas não vira unidade armazenada nem evidência; sem migration.

Related #1395

docs/changelog/2026-09-30-c237.md

@@ -0,0 +1 @@
+**C237 — Acervo por tema: o vetor da fala passa a incluir o sumário/keywords oficiais (2026-09-30):** a Fase 5 pós-deploy do C229 mediu o corpus real e o aceite "impeachment → fala da votação de Dilma" falhava: a 222 (17/04/2016) ficava em #841 e a 216 (14/04/2016) em #82, enquanto 8 das 10 primeiras eram falas que sequer citam impeachment/impedimento (golpe, cassação, PGR/Temer) — o índice só embutia o ASR, e a 222 nunca pronuncia "impeachment"; a relevância dela mora no **sumário/keywords oficiais da Câmara**, que nunca entravam no vetor. Fix: `planSpeechIndex` ganha `metadataText` (sumário — ou título no web — + keywords cruas; `speechMetadataText`) e o **hash de idempotência inclui a metadata** (editar o sumário reindexa); o vetor da fala passa a ser `normalize(média_das_unidades + 2·metadata)` (`composeSpeechVector`, `SPEECH_METADATA_VECTOR_WEIGHT = 2` calibrado no corpus real: w=2 leva a 222 à #8 com o cluster do tema na frente; w=1 deixava em #25); a metadata é vetorizada no mesmo lote do provedor mas **não vira unidade armazenada nem evidência** (o "Trecho mais próximo do tema" continua falado) e não há migration. CLI `acervo:index` seleciona/passa `title`/`keywords` e o relatório ganha `(+N metadados)` ao lado dos trechos (dry-run e write), com testes unit do plano/hash/composição/relatório e int do ranking (fala cujo ASR não cita o tema vence pela metadata). Reindexar produção após o deploy (hash muda para toda fala com metadata; ~US$ 0,007 e ~31 min no corpus atual). Evidência da Fase 5 e aceite na Issue #1395.

scripts/index-speeches.mjs

@@ -80,7 +80,14 @@ const loadSources = async (payload, { source, limit }) => {
     depth: 0,
     ...(limit ? { limit, page: 1 } : { limit: 0, pagination: false }),
     sort: 'speechAt',
-    select: { sourceKey: true, officialTranscript: true, summary: true, searchText: true },
+    select: {
+      sourceKey: true,
+      officialTranscript: true,
+      summary: true,
+      title: true,
+      keywords: true,
+      searchText: true,
+    },
     // Intentional bypass: the index CLI is a trusted actor with no session.
     overrideAccess: true,
   })
@@ -116,6 +123,8 @@ const loadSources = async (payload, { source, limit }) => {
     sourceKey: speech.sourceKey,
     officialTranscript: speech.officialTranscript,
     summary: speech.summary,
+    title: speech.title,
+    keywords: speech.keywords,
     searchText: speech.searchText,
     segments: segmentsBySpeech.get(speech.id) ?? [],
   }))
@@ -204,6 +213,7 @@ async function main() {
     indexed: result.indexed,
     failed: result.failed,
     units: result.units,
+    metadataTexts: result.metadataTexts,
     estimatedBytes: speechIndexEstimatedBytes(result.units, DEEPINFRA_EMBED_DIMENSIONS),
     promptTokens: result.promptTokens,
     costUsd: speechIndexCostUsd(result.promptTokens, DEEPINFRA_EMBED_COST_PER_MILLION_TOKENS_USD),

scripts/lib/speechIndex.mjs

@@ -71,11 +71,16 @@ export const speechIndexCostUsd = (promptTokens, pricePerMillionTokensUsd) =>
 /** Rough JSON cost of the vectors: units × dims × ~8 bytes per number. */
 export const speechIndexEstimatedBytes = (units, dimensions) => units * dimensions * 8
 
+/** C237 — the official metadata rides the same provider call but is not a stored unit. */
+const metadataSuffix = (metadataTexts) =>
+  typeof metadataTexts === 'number' && metadataTexts > 0 ? ` (+${metadataTexts} metadados)` : ''
+
 /**
  * @param {{
  *   target: string, mode: 'index' | 'dry-run', source: string,
  *   speeches: number, upToDate: number, skippedNoText: number, toIndex: number,
- *   units: number, estimatedBytes?: number, indexed?: number, failed?: number,
+ *   units: number, metadataTexts?: number, estimatedBytes?: number,
+ *   indexed?: number, failed?: number,
  *   promptTokens?: number, costUsd?: number, durationMs: number,
  * }} report
  * @returns {string[]}
@@ -85,6 +90,7 @@ export const formatSpeechIndexReport = (report) => {
     typeof report.estimatedBytes === 'number'
       ? ` | bytes estimados: ${(report.estimatedBytes / 1_000_000).toFixed(1)} MB`
       : ''
+  const metadata = metadataSuffix(report.metadataTexts)
   const lines = [
     `[acervo:index] alvo: ${report.target} | modo: ${report.mode} | fonte: ${report.source}`,
     `falas: ${report.speeches} | atualizadas: ${report.upToDate} | sem texto: ${report.skippedNoText} | a indexar: ${report.toIndex}`,
@@ -92,10 +98,10 @@ export const formatSpeechIndexReport = (report) => {
 
   if (report.mode === 'index') {
     lines.push(
-      `indexadas: ${report.indexed ?? 0} | falhas: ${report.failed ?? 0} | trechos vetorizados: ${report.units}${bytes} | tokens: ${report.promptTokens ?? 0} | custo estimado: US$ ${(report.costUsd ?? 0).toFixed(4)}`,
+      `indexadas: ${report.indexed ?? 0} | falhas: ${report.failed ?? 0} | trechos vetorizados: ${report.units}${metadata}${bytes} | tokens: ${report.promptTokens ?? 0} | custo estimado: US$ ${(report.costUsd ?? 0).toFixed(4)}`,
     )
   } else {
-    lines.push(`trechos a vetorizar: ${report.units}${bytes}`)
+    lines.push(`trechos a vetorizar: ${report.units}${metadata}${bytes}`)
   }
 
   lines.push(`duração: ${(report.durationMs / 1000).toFixed(1)}s`)

src/lib/speechSemantic.ts

@@ -11,6 +11,16 @@
 /** The provisional cut-off of the theme mode — calibrated by `acervo:index --probe`. */
 export const SPEECH_SEMANTIC_MIN_COSINE = 0.4
 
+/**
+ * C237 — weight of the official metadata (summary/title + keywords) against the
+ * mean of the spoken units when composing the speech-level vector. Calibrated
+ * on the production corpus (2026-09-30): with weight 1 the Dilma impeachment
+ * vote speech (222) sat at #25 for the theme "impeachment"; with 2 it reaches
+ * the first page (#8) and the topic cluster (2016 process + Bolsonaro requests)
+ * leads, without dropping the other acceptance themes out of their cluster.
+ */
+export const SPEECH_METADATA_VECTOR_WEIGHT = 2
+
 /** Text windows of speeches without ASR segments: ~1200 chars, ~200 of overlap. */
 const SPEECH_WINDOW_SIZE_CHARS = 1200
 const SPEECH_WINDOW_OVERLAP_CHARS = 200
@@ -239,10 +249,50 @@ type SpeechIndexUnit = {
 
 export type SpeechIndexPlan = {
   units: SpeechIndexUnit[]
-  /** Hash of every unit text + model: the CLI's skip key. */
+  /**
+   * C237 — official metadata text folded into the speech-level vector (never
+   * stored as a unit, never evidence). `null` when the speech carries none.
+   */
+  metadataText: string | null
+  /** Hash of every unit text + metadata + model: the CLI's skip key. */
   speechHash: string
 }
 
+/**
+ * C237 — the official metadata of one speech: the Câmara summary (the web title
+ * when there is no summary) plus the raw official keywords. This is the text
+ * that states the *topic* even when the spoken passage never names it (the
+ * impeachment vote speech never says "impeachment").
+ */
+export const speechMetadataText = (speech: {
+  summary?: string | null
+  title?: string | null
+  keywords?: readonly string[] | null
+}): string | null => {
+  const head = [speech.summary, speech.title].find(
+    (candidate) => typeof candidate === 'string' && candidate.trim() !== '',
+  )
+  const keywords = (speech.keywords ?? []).map((keyword) => keyword.trim()).filter(Boolean)
+  const text = [head?.trim() ?? '', keywords.join(', ')].filter(Boolean).join('\n')
+  return text === '' ? null : text
+}
+
+/**
+ * C237 — the speech-level vector of the index: the normalized mean of the unit
+ * vectors plus the weighted official metadata. A speech without metadata keeps
+ * the pre-C237 mean exactly.
+ */
+export const composeSpeechVector = (
+  unitVectors: readonly SpeechSemanticVector[],
+  metadataVector: SpeechSemanticVector | null,
+): number[] => {
+  const mean = meanPoolVectors(unitVectors)
+  if (!metadataVector || metadataVector.length !== mean.length || mean.length === 0) return mean
+  return normalizeVector(
+    mean.map((value, index) => value + SPEECH_METADATA_VECTOR_WEIGHT * metadataVector[index]),
+  )
+}
+
 export type SpeechIndexSegment = {
   order: number
   startSeconds: number
@@ -259,6 +309,8 @@ export const planSpeechIndex = (
   speech: {
     officialTranscript?: string | null
     summary?: string | null
+    title?: string | null
+    keywords?: readonly string[] | null
     searchText?: string | null
     segments?: readonly SpeechIndexSegment[]
   },
@@ -289,9 +341,13 @@ export const planSpeechIndex = (
 
   if (units.length === 0) return null
 
+  const metadataText = speechMetadataText(speech)
   const speechHash = speechEmbeddingHash(
-    units.map((unit) => `${unit.kind}:${unit.order}:${unit.text}`).join('\n'),
+    [
+      ...units.map((unit) => `${unit.kind}:${unit.order}:${unit.text}`),
+      ...(metadataText ? [`metadata:${metadataText}`] : []),
+    ].join('\n'),
     model,
   )
-  return { units, speechHash }
+  return { units, metadataText, speechHash }
 }

src/utilities/speech/speechEmbeddingIndex.ts

@@ -3,8 +3,8 @@ import 'server-only'
 import type { Payload } from 'payload'
 
 import {
+  composeSpeechVector,
   cosineSimilarity,
-  meanPoolVectors,
   planSpeechIndex,
   type SpeechIndexPlan,
   type SpeechIndexSegment,
@@ -24,6 +24,10 @@ export type SpeechIndexSource = {
   id: number
   officialTranscript?: string | null
   summary?: string | null
+  /** C237 — web title, the metadata head when there is no summary. */
+  title?: string | null
+  /** C237 — raw official keywords, folded into the metadata text. */
+  keywords?: readonly string[] | null
   searchText?: string | null
   segments: readonly SpeechIndexSegment[]
 }
@@ -41,6 +45,8 @@ export type SpeechIndexPlanBatch = {
   /** No usable text at all. */
   skippedNoText: number
   units: number
+  /** C237 — one extra provider input per stale speech that has metadata. */
+  metadataTexts: number
 }
 
 export type SpeechIndexBatchResult = {
@@ -48,7 +54,10 @@ export type SpeechIndexBatchResult = {
   upToDate: number
   skippedNoText: number
   failed: number
+  /** Stored retrieval units (segments/windows). */
   units: number
+  /** C237 — metadata texts embedded on top of the units (not stored). */
+  metadataTexts: number
   promptTokens: number
 }
 
@@ -120,6 +129,7 @@ export const planSpeechIndexBatch = async (
     upToDate: planned.length - stale.length,
     skippedNoText,
     units: stale.reduce((total, { plan }) => total + plan.units.length, 0),
+    metadataTexts: stale.reduce((total, { plan }) => total + (plan.metadataText ? 1 : 0), 0),
   }
 }
 
@@ -141,29 +151,41 @@ export const indexSpeechSources = async (
     upToDate: batch.upToDate,
     skippedNoText: batch.skippedNoText,
     units: 0,
+    metadataTexts: 0,
     promptTokens: 0,
   }
   if (batch.stale.length === 0) return { ...base, indexed: 0, failed: 0 }
 
-  const texts = batch.stale.flatMap(({ plan }) => plan.units.map((unit) => unit.text))
+  // C237 — the official metadata rides the same ordered provider call, right
+  // after the units of each speech; it feeds the speech vector only.
+  const texts = batch.stale.flatMap(({ plan }) => [
+    ...plan.units.map((unit) => unit.text),
+    ...(plan.metadataText ? [plan.metadataText] : []),
+  ])
   const embedded = await embedTexts(texts)
   if (!embedded) {
     return { ...base, indexed: 0, failed: batch.stale.length }
   }
   // The whole batch was embedded already: the tokens were spent even if a
   // later write fails.
-  const spent = { units: texts.length, promptTokens: embedded.promptTokens }
+  const spent = {
+    units: batch.units,
+    metadataTexts: batch.metadataTexts,
+    promptTokens: embedded.promptTokens,
+  }
 
   let cursor = 0
   let indexed = 0
   for (const { source, plan } of batch.stale) {
     const vectors = embedded.vectors.slice(cursor, cursor + plan.units.length)
     cursor += plan.units.length
+    const metadataVector = plan.metadataText ? (embedded.vectors[cursor] ?? null) : null
+    if (plan.metadataText) cursor += 1
     if (vectors.length !== plan.units.length) {
       return { ...base, ...spent, indexed, failed: batch.stale.length - indexed }
     }
 
-    const speechVector = meanPoolVectors(vectors)
+    const speechVector = composeSpeechVector(vectors, metadataVector)
     await withPayloadTransaction(payload, async ({ req }) => {
       await payload.delete({
         collection: 'speechEmbedding',
@@ -299,6 +321,7 @@ export type SpeechIndexPreview = {
   skippedNoText: number
   toIndex: number
   units: number
+  metadataTexts: number
 }
 
 export const previewSpeechIndex = (
@@ -310,4 +333,5 @@ export const previewSpeechIndex = (
   skippedNoText: batch.skippedNoText,
   toIndex: batch.stale.length,
   units: batch.units,
+  metadataTexts: batch.metadataTexts,
 })

tests/int/speechAcervo.int.spec.ts

@@ -613,6 +613,70 @@ describe('speech acervo (C154)', () => {
     ).toContain(`Primeiro trecho ${runId}`)
   })
 
+  it('folds the official metadata into the speech vector (C237)', async () => {
+    const runId = randomUUID().slice(0, 8)
+    const offTopic = [0.5, Math.sqrt(3) / 2]
+    const summary = `Processo de impeachment da Presidenta Dilma Rousseff ${runId}`
+    const idWithMetadata = await createSpeech({
+      speechAt: '1988-01-01T10:00',
+      year: 1988,
+      summary,
+      keywords: ['IMPEACHMENT'],
+      segments: [{ startSeconds: 0, endSeconds: 3, text: `Trecho sem o tema ${runId}` }],
+    })
+    const idWithout = await createSpeech({
+      speechAt: '1988-01-02T10:00',
+      year: 1988,
+      segments: [{ startSeconds: 0, endSeconds: 3, text: `Trecho sem o tema ${runId}` }],
+    })
+    const embedCalls: string[][] = []
+    const embedTexts = async (texts: readonly string[]) => {
+      embedCalls.push([...texts])
+      return {
+        vectors: texts.map((text) =>
+          text.includes(runId) && text.startsWith('Processo') ? [1, 0] : offTopic,
+        ),
+        promptTokens: 5,
+      }
+    }
+
+    const result = await indexSpeechSources(
+      payload,
+      [
+        {
+          id: idWithMetadata,
+          summary,
+          keywords: ['IMPEACHMENT'],
+          segments: [{ order: 1, startSeconds: 0, text: `Trecho sem o tema ${runId}` }],
+        },
+        {
+          id: idWithout,
+          segments: [{ order: 1, startSeconds: 0, text: `Trecho sem o tema ${runId}` }],
+        },
+      ],
+      { embedTexts },
+    )
+
+    expect(result).toMatchObject({ indexed: 2, failed: 0, units: 2, metadataTexts: 1 })
+    expect(embedCalls[0]).toEqual([
+      `Trecho sem o tema ${runId}`,
+      `${summary}\nIMPEACHMENT`,
+      `Trecho sem o tema ${runId}`,
+    ])
+
+    const { coordinator } = await createUsers()
+    const data = await loadSpeechAcervoPageData(
+      payload,
+      coordinator,
+      { q: `tema ${runId}`, mode: 'tema', year: '1988' },
+      async () => [1, 0],
+    )
+    // Both passages never state the theme; the official metadata is what ranks
+    // the speech that carries it above the identical spoken-only neighbour.
+    expect(data.rows.map((row) => row.id)).toEqual([idWithMetadata, idWithout])
+    expect(data.rows[0]?.semanticMatch).toBe(true)
+  })
+
   it('cascades the index rows when the speech is deleted (C229)', async () => {
     const id = await createSpeech()
     await seedSpeechEmbedding(id, 'speech', null, [1, 0])

tests/unit/speechIndex.unit.spec.ts

@@ -112,6 +112,27 @@ describe('formatSpeechIndexReport', () => {
     expect(lines[2]).toContain('tokens: 123456')
     expect(lines[2]).toContain('custo estimado: US$ 0.0012')
   })
+
+  it('reports the metadata inputs on top of the stored units (C237)', () => {
+    const dryRun = formatSpeechIndexReport({ ...base, mode: 'dry-run', metadataTexts: 42 })
+    expect(dryRun[2]).toContain('trechos a vetorizar: 300 (+42 metadados)')
+
+    const write = formatSpeechIndexReport({
+      ...base,
+      mode: 'index',
+      metadataTexts: 42,
+      indexed: 15,
+      failed: 0,
+      promptTokens: 10,
+      costUsd: 0,
+    })
+    expect(write[2]).toContain('trechos vetorizados: 300 (+42 metadados)')
+
+    // No metadata (speech without summary/title/keywords) keeps the old shape.
+    expect(formatSpeechIndexReport({ ...base, mode: 'dry-run' })[2]).toBe(
+      'trechos a vetorizar: 300',
+    )
+  })
 })
 
 describe('formatSpeechIndexProbe', () => {

tests/unit/speechSemantic.unit.spec.ts

@@ -3,6 +3,7 @@
 import { describe, expect, it } from 'vitest'
 
 import {
+  composeSpeechVector,
   cosineSimilarity,
   dotProduct,
   meanPoolVectors,
@@ -11,8 +12,10 @@ import {
   planSpeechIndex,
   rankSemanticHits,
   sortSemanticHits,
+  SPEECH_METADATA_VECTOR_WEIGHT,
   SPEECH_SEMANTIC_MIN_COSINE,
   speechEmbeddingHash,
+  speechMetadataText,
   speechSemanticSourceText,
   speechTextWindows,
   type SpeechSemanticCandidate,
@@ -246,4 +249,65 @@ describe('planSpeechIndex', () => {
       planSpeechIndex(speech, { model: 'b' })?.speechHash,
     )
   })
+
+  it('plans the official metadata and pins it in the hash (C237)', () => {
+    const speech = {
+      summary: 'Processo de impeachment da Presidenta Dilma Rousseff.',
+      keywords: ['IMPEACHMENT', 'DILMA ROUSSEFF'],
+      segments: [{ order: 1, startSeconds: 0, text: 'Sr. Presidente...' }],
+    }
+    const plan = planSpeechIndex(speech, { model })
+
+    expect(plan?.metadataText).toBe(
+      'Processo de impeachment da Presidenta Dilma Rousseff.\nIMPEACHMENT, DILMA ROUSSEFF',
+    )
+    expect(plan?.speechHash).toBe(
+      speechEmbeddingHash(
+        'segment:1:Sr. Presidente...\nmetadata:Processo de impeachment da Presidenta Dilma Rousseff.\nIMPEACHMENT, DILMA ROUSSEFF',
+        model,
+      ),
+    )
+    // Editing the summary or the keywords must reindex the speech.
+    expect(
+      planSpeechIndex({ ...speech, summary: 'Outro sumário' }, { model })?.speechHash,
+    ).not.toBe(plan?.speechHash)
+    expect(planSpeechIndex({ ...speech, keywords: ['OUTRA'] }, { model })?.speechHash).not.toBe(
+      plan?.speechHash,
+    )
+    // A speech without metadata keeps the pre-C237 hash bytes.
+    expect(planSpeechIndex({ segments: speech.segments }, { model })?.metadataText).toBeNull()
+    expect(planSpeechIndex({ segments: speech.segments }, { model })?.speechHash).toBe(
+      speechEmbeddingHash('segment:1:Sr. Presidente...', model),
+    )
+  })
+})
+
+describe('speechMetadataText (C237)', () => {
+  it('prefers the summary over the title and joins the raw keywords', () => {
+    expect(
+      speechMetadataText({ summary: 'Resumo', title: 'Título', keywords: ['SUS', ' Saúde '] }),
+    ).toBe('Resumo\nSUS, Saúde')
+    expect(speechMetadataText({ title: 'Título', keywords: [] })).toBe('Título')
+    expect(speechMetadataText({ summary: '  ', keywords: ['   '] })).toBeNull()
+    expect(speechMetadataText({})).toBeNull()
+  })
+})
+
+describe('composeSpeechVector (C237)', () => {
+  const unit = [0.6, 0.8]
+
+  it('keeps the pure mean without metadata', () => {
+    expect(composeSpeechVector([unit], null)).toEqual(meanPoolVectors([unit]))
+    // A metadata vector of another dimension never poisons the mean.
+    expect(composeSpeechVector([unit], [1, 0, 0])).toEqual(meanPoolVectors([unit]))
+  })
+
+  it('pulls the speech vector toward the weighted metadata', () => {
+    const composed = composeSpeechVector([unit], [1, 0])
+
+    expect(composed).toEqual(
+      normalizeVector([unit[0] + SPEECH_METADATA_VECTOR_WEIGHT * 1, unit[1]]),
+    )
+    expect(cosineSimilarity(composed, [1, 0])).toBeGreaterThan(cosineSimilarity(unit, [1, 0]))
+  })
 })