mirror of
https://github.com/fsolla/teqo.git
synced 2026-11-21 20:03:08 +00:00
feat(C237): vetor da fala inclui o sumário/keywords oficiais
A Fase 5 pós-deploy do C229 mediu o corpus real e o aceite 'impeachment' falhava: a 222 (votação do impeachment de Dilma, 17/04/2016) ficava em #841 porque o índice só embutia o ASR — e a fala nunca pronuncia 'impeachment'; a relevância dela mora no sumário/keywords oficiais da Câmara. O plano do índice passa a carregar metadataText (sumário — ou título no web — + keywords, incluído no hash de idempotência) e o vetor da fala vira normalize(média das unidades + 2·metadata), calibrado no corpus real (w=2: 222 em #8; w=1: #25). A metadata é vetorizada no mesmo lote do provedor mas não vira unidade armazenada nem evidência; sem migration. Related #1395
This commit is contained in:
parent
5eb8fd75ef
commit
3201b7a93f
8 changed files with 257 additions and 11 deletions
1
docs/changelog/2026-09-30-c237.md
Normal file
1
docs/changelog/2026-09-30-c237.md
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
**C237 — Acervo por tema: o vetor da fala passa a incluir o sumário/keywords oficiais (2026-09-30):** a Fase 5 pós-deploy do C229 mediu o corpus real e o aceite "impeachment → fala da votação de Dilma" falhava: a 222 (17/04/2016) ficava em #841 e a 216 (14/04/2016) em #82, enquanto 8 das 10 primeiras eram falas que sequer citam impeachment/impedimento (golpe, cassação, PGR/Temer) — o índice só embutia o ASR, e a 222 nunca pronuncia "impeachment"; a relevância dela mora no **sumário/keywords oficiais da Câmara**, que nunca entravam no vetor. Fix: `planSpeechIndex` ganha `metadataText` (sumário — ou título no web — + keywords cruas; `speechMetadataText`) e o **hash de idempotência inclui a metadata** (editar o sumário reindexa); o vetor da fala passa a ser `normalize(média_das_unidades + 2·metadata)` (`composeSpeechVector`, `SPEECH_METADATA_VECTOR_WEIGHT = 2` calibrado no corpus real: w=2 leva a 222 à #8 com o cluster do tema na frente; w=1 deixava em #25); a metadata é vetorizada no mesmo lote do provedor mas **não vira unidade armazenada nem evidência** (o "Trecho mais próximo do tema" continua falado) e não há migration. CLI `acervo:index` seleciona/passa `title`/`keywords` e o relatório ganha `(+N metadados)` ao lado dos trechos (dry-run e write), com testes unit do plano/hash/composição/relatório e int do ranking (fala cujo ASR não cita o tema vence pela metadata). Reindexar produção após o deploy (hash muda para toda fala com metadata; ~US$ 0,007 e ~31 min no corpus atual). Evidência da Fase 5 e aceite na Issue #1395.
|
||||||
|
|
@ -80,7 +80,14 @@ const loadSources = async (payload, { source, limit }) => {
|
||||||
depth: 0,
|
depth: 0,
|
||||||
...(limit ? { limit, page: 1 } : { limit: 0, pagination: false }),
|
...(limit ? { limit, page: 1 } : { limit: 0, pagination: false }),
|
||||||
sort: 'speechAt',
|
sort: 'speechAt',
|
||||||
select: { sourceKey: true, officialTranscript: true, summary: true, searchText: true },
|
select: {
|
||||||
|
sourceKey: true,
|
||||||
|
officialTranscript: true,
|
||||||
|
summary: true,
|
||||||
|
title: true,
|
||||||
|
keywords: true,
|
||||||
|
searchText: true,
|
||||||
|
},
|
||||||
// Intentional bypass: the index CLI is a trusted actor with no session.
|
// Intentional bypass: the index CLI is a trusted actor with no session.
|
||||||
overrideAccess: true,
|
overrideAccess: true,
|
||||||
})
|
})
|
||||||
|
|
@ -116,6 +123,8 @@ const loadSources = async (payload, { source, limit }) => {
|
||||||
sourceKey: speech.sourceKey,
|
sourceKey: speech.sourceKey,
|
||||||
officialTranscript: speech.officialTranscript,
|
officialTranscript: speech.officialTranscript,
|
||||||
summary: speech.summary,
|
summary: speech.summary,
|
||||||
|
title: speech.title,
|
||||||
|
keywords: speech.keywords,
|
||||||
searchText: speech.searchText,
|
searchText: speech.searchText,
|
||||||
segments: segmentsBySpeech.get(speech.id) ?? [],
|
segments: segmentsBySpeech.get(speech.id) ?? [],
|
||||||
}))
|
}))
|
||||||
|
|
@ -204,6 +213,7 @@ async function main() {
|
||||||
indexed: result.indexed,
|
indexed: result.indexed,
|
||||||
failed: result.failed,
|
failed: result.failed,
|
||||||
units: result.units,
|
units: result.units,
|
||||||
|
metadataTexts: result.metadataTexts,
|
||||||
estimatedBytes: speechIndexEstimatedBytes(result.units, DEEPINFRA_EMBED_DIMENSIONS),
|
estimatedBytes: speechIndexEstimatedBytes(result.units, DEEPINFRA_EMBED_DIMENSIONS),
|
||||||
promptTokens: result.promptTokens,
|
promptTokens: result.promptTokens,
|
||||||
costUsd: speechIndexCostUsd(result.promptTokens, DEEPINFRA_EMBED_COST_PER_MILLION_TOKENS_USD),
|
costUsd: speechIndexCostUsd(result.promptTokens, DEEPINFRA_EMBED_COST_PER_MILLION_TOKENS_USD),
|
||||||
|
|
|
||||||
|
|
@ -71,11 +71,16 @@ export const speechIndexCostUsd = (promptTokens, pricePerMillionTokensUsd) =>
|
||||||
/** Rough JSON cost of the vectors: units × dims × ~8 bytes per number. */
|
/** Rough JSON cost of the vectors: units × dims × ~8 bytes per number. */
|
||||||
export const speechIndexEstimatedBytes = (units, dimensions) => units * dimensions * 8
|
export const speechIndexEstimatedBytes = (units, dimensions) => units * dimensions * 8
|
||||||
|
|
||||||
|
/** C237 — the official metadata rides the same provider call but is not a stored unit. */
|
||||||
|
const metadataSuffix = (metadataTexts) =>
|
||||||
|
typeof metadataTexts === 'number' && metadataTexts > 0 ? ` (+${metadataTexts} metadados)` : ''
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {{
|
* @param {{
|
||||||
* target: string, mode: 'index' | 'dry-run', source: string,
|
* target: string, mode: 'index' | 'dry-run', source: string,
|
||||||
* speeches: number, upToDate: number, skippedNoText: number, toIndex: number,
|
* speeches: number, upToDate: number, skippedNoText: number, toIndex: number,
|
||||||
* units: number, estimatedBytes?: number, indexed?: number, failed?: number,
|
* units: number, metadataTexts?: number, estimatedBytes?: number,
|
||||||
|
* indexed?: number, failed?: number,
|
||||||
* promptTokens?: number, costUsd?: number, durationMs: number,
|
* promptTokens?: number, costUsd?: number, durationMs: number,
|
||||||
* }} report
|
* }} report
|
||||||
* @returns {string[]}
|
* @returns {string[]}
|
||||||
|
|
@ -85,6 +90,7 @@ export const formatSpeechIndexReport = (report) => {
|
||||||
typeof report.estimatedBytes === 'number'
|
typeof report.estimatedBytes === 'number'
|
||||||
? ` | bytes estimados: ${(report.estimatedBytes / 1_000_000).toFixed(1)} MB`
|
? ` | bytes estimados: ${(report.estimatedBytes / 1_000_000).toFixed(1)} MB`
|
||||||
: ''
|
: ''
|
||||||
|
const metadata = metadataSuffix(report.metadataTexts)
|
||||||
const lines = [
|
const lines = [
|
||||||
`[acervo:index] alvo: ${report.target} | modo: ${report.mode} | fonte: ${report.source}`,
|
`[acervo:index] alvo: ${report.target} | modo: ${report.mode} | fonte: ${report.source}`,
|
||||||
`falas: ${report.speeches} | atualizadas: ${report.upToDate} | sem texto: ${report.skippedNoText} | a indexar: ${report.toIndex}`,
|
`falas: ${report.speeches} | atualizadas: ${report.upToDate} | sem texto: ${report.skippedNoText} | a indexar: ${report.toIndex}`,
|
||||||
|
|
@ -92,10 +98,10 @@ export const formatSpeechIndexReport = (report) => {
|
||||||
|
|
||||||
if (report.mode === 'index') {
|
if (report.mode === 'index') {
|
||||||
lines.push(
|
lines.push(
|
||||||
`indexadas: ${report.indexed ?? 0} | falhas: ${report.failed ?? 0} | trechos vetorizados: ${report.units}${bytes} | tokens: ${report.promptTokens ?? 0} | custo estimado: US$ ${(report.costUsd ?? 0).toFixed(4)}`,
|
`indexadas: ${report.indexed ?? 0} | falhas: ${report.failed ?? 0} | trechos vetorizados: ${report.units}${metadata}${bytes} | tokens: ${report.promptTokens ?? 0} | custo estimado: US$ ${(report.costUsd ?? 0).toFixed(4)}`,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
lines.push(`trechos a vetorizar: ${report.units}${bytes}`)
|
lines.push(`trechos a vetorizar: ${report.units}${metadata}${bytes}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
lines.push(`duração: ${(report.durationMs / 1000).toFixed(1)}s`)
|
lines.push(`duração: ${(report.durationMs / 1000).toFixed(1)}s`)
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,16 @@
|
||||||
/** The provisional cut-off of the theme mode — calibrated by `acervo:index --probe`. */
|
/** The provisional cut-off of the theme mode — calibrated by `acervo:index --probe`. */
|
||||||
export const SPEECH_SEMANTIC_MIN_COSINE = 0.4
|
export const SPEECH_SEMANTIC_MIN_COSINE = 0.4
|
||||||
|
|
||||||
|
/**
|
||||||
|
* C237 — weight of the official metadata (summary/title + keywords) against the
|
||||||
|
* mean of the spoken units when composing the speech-level vector. Calibrated
|
||||||
|
* on the production corpus (2026-09-30): with weight 1 the Dilma impeachment
|
||||||
|
* vote speech (222) sat at #25 for the theme "impeachment"; with 2 it reaches
|
||||||
|
* the first page (#8) and the topic cluster (2016 process + Bolsonaro requests)
|
||||||
|
* leads, without dropping the other acceptance themes out of their cluster.
|
||||||
|
*/
|
||||||
|
export const SPEECH_METADATA_VECTOR_WEIGHT = 2
|
||||||
|
|
||||||
/** Text windows of speeches without ASR segments: ~1200 chars, ~200 of overlap. */
|
/** Text windows of speeches without ASR segments: ~1200 chars, ~200 of overlap. */
|
||||||
const SPEECH_WINDOW_SIZE_CHARS = 1200
|
const SPEECH_WINDOW_SIZE_CHARS = 1200
|
||||||
const SPEECH_WINDOW_OVERLAP_CHARS = 200
|
const SPEECH_WINDOW_OVERLAP_CHARS = 200
|
||||||
|
|
@ -239,10 +249,50 @@ type SpeechIndexUnit = {
|
||||||
|
|
||||||
export type SpeechIndexPlan = {
|
export type SpeechIndexPlan = {
|
||||||
units: SpeechIndexUnit[]
|
units: SpeechIndexUnit[]
|
||||||
/** Hash of every unit text + model: the CLI's skip key. */
|
/**
|
||||||
|
* C237 — official metadata text folded into the speech-level vector (never
|
||||||
|
* stored as a unit, never evidence). `null` when the speech carries none.
|
||||||
|
*/
|
||||||
|
metadataText: string | null
|
||||||
|
/** Hash of every unit text + metadata + model: the CLI's skip key. */
|
||||||
speechHash: string
|
speechHash: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* C237 — the official metadata of one speech: the Câmara summary (the web title
|
||||||
|
* when there is no summary) plus the raw official keywords. This is the text
|
||||||
|
* that states the *topic* even when the spoken passage never names it (the
|
||||||
|
* impeachment vote speech never says "impeachment").
|
||||||
|
*/
|
||||||
|
export const speechMetadataText = (speech: {
|
||||||
|
summary?: string | null
|
||||||
|
title?: string | null
|
||||||
|
keywords?: readonly string[] | null
|
||||||
|
}): string | null => {
|
||||||
|
const head = [speech.summary, speech.title].find(
|
||||||
|
(candidate) => typeof candidate === 'string' && candidate.trim() !== '',
|
||||||
|
)
|
||||||
|
const keywords = (speech.keywords ?? []).map((keyword) => keyword.trim()).filter(Boolean)
|
||||||
|
const text = [head?.trim() ?? '', keywords.join(', ')].filter(Boolean).join('\n')
|
||||||
|
return text === '' ? null : text
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* C237 — the speech-level vector of the index: the normalized mean of the unit
|
||||||
|
* vectors plus the weighted official metadata. A speech without metadata keeps
|
||||||
|
* the pre-C237 mean exactly.
|
||||||
|
*/
|
||||||
|
export const composeSpeechVector = (
|
||||||
|
unitVectors: readonly SpeechSemanticVector[],
|
||||||
|
metadataVector: SpeechSemanticVector | null,
|
||||||
|
): number[] => {
|
||||||
|
const mean = meanPoolVectors(unitVectors)
|
||||||
|
if (!metadataVector || metadataVector.length !== mean.length || mean.length === 0) return mean
|
||||||
|
return normalizeVector(
|
||||||
|
mean.map((value, index) => value + SPEECH_METADATA_VECTOR_WEIGHT * metadataVector[index]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
export type SpeechIndexSegment = {
|
export type SpeechIndexSegment = {
|
||||||
order: number
|
order: number
|
||||||
startSeconds: number
|
startSeconds: number
|
||||||
|
|
@ -259,6 +309,8 @@ export const planSpeechIndex = (
|
||||||
speech: {
|
speech: {
|
||||||
officialTranscript?: string | null
|
officialTranscript?: string | null
|
||||||
summary?: string | null
|
summary?: string | null
|
||||||
|
title?: string | null
|
||||||
|
keywords?: readonly string[] | null
|
||||||
searchText?: string | null
|
searchText?: string | null
|
||||||
segments?: readonly SpeechIndexSegment[]
|
segments?: readonly SpeechIndexSegment[]
|
||||||
},
|
},
|
||||||
|
|
@ -289,9 +341,13 @@ export const planSpeechIndex = (
|
||||||
|
|
||||||
if (units.length === 0) return null
|
if (units.length === 0) return null
|
||||||
|
|
||||||
|
const metadataText = speechMetadataText(speech)
|
||||||
const speechHash = speechEmbeddingHash(
|
const speechHash = speechEmbeddingHash(
|
||||||
units.map((unit) => `${unit.kind}:${unit.order}:${unit.text}`).join('\n'),
|
[
|
||||||
|
...units.map((unit) => `${unit.kind}:${unit.order}:${unit.text}`),
|
||||||
|
...(metadataText ? [`metadata:${metadataText}`] : []),
|
||||||
|
].join('\n'),
|
||||||
model,
|
model,
|
||||||
)
|
)
|
||||||
return { units, speechHash }
|
return { units, metadataText, speechHash }
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,8 @@ import 'server-only'
|
||||||
import type { Payload } from 'payload'
|
import type { Payload } from 'payload'
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
composeSpeechVector,
|
||||||
cosineSimilarity,
|
cosineSimilarity,
|
||||||
meanPoolVectors,
|
|
||||||
planSpeechIndex,
|
planSpeechIndex,
|
||||||
type SpeechIndexPlan,
|
type SpeechIndexPlan,
|
||||||
type SpeechIndexSegment,
|
type SpeechIndexSegment,
|
||||||
|
|
@ -24,6 +24,10 @@ export type SpeechIndexSource = {
|
||||||
id: number
|
id: number
|
||||||
officialTranscript?: string | null
|
officialTranscript?: string | null
|
||||||
summary?: string | null
|
summary?: string | null
|
||||||
|
/** C237 — web title, the metadata head when there is no summary. */
|
||||||
|
title?: string | null
|
||||||
|
/** C237 — raw official keywords, folded into the metadata text. */
|
||||||
|
keywords?: readonly string[] | null
|
||||||
searchText?: string | null
|
searchText?: string | null
|
||||||
segments: readonly SpeechIndexSegment[]
|
segments: readonly SpeechIndexSegment[]
|
||||||
}
|
}
|
||||||
|
|
@ -41,6 +45,8 @@ export type SpeechIndexPlanBatch = {
|
||||||
/** No usable text at all. */
|
/** No usable text at all. */
|
||||||
skippedNoText: number
|
skippedNoText: number
|
||||||
units: number
|
units: number
|
||||||
|
/** C237 — one extra provider input per stale speech that has metadata. */
|
||||||
|
metadataTexts: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SpeechIndexBatchResult = {
|
export type SpeechIndexBatchResult = {
|
||||||
|
|
@ -48,7 +54,10 @@ export type SpeechIndexBatchResult = {
|
||||||
upToDate: number
|
upToDate: number
|
||||||
skippedNoText: number
|
skippedNoText: number
|
||||||
failed: number
|
failed: number
|
||||||
|
/** Stored retrieval units (segments/windows). */
|
||||||
units: number
|
units: number
|
||||||
|
/** C237 — metadata texts embedded on top of the units (not stored). */
|
||||||
|
metadataTexts: number
|
||||||
promptTokens: number
|
promptTokens: number
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -120,6 +129,7 @@ export const planSpeechIndexBatch = async (
|
||||||
upToDate: planned.length - stale.length,
|
upToDate: planned.length - stale.length,
|
||||||
skippedNoText,
|
skippedNoText,
|
||||||
units: stale.reduce((total, { plan }) => total + plan.units.length, 0),
|
units: stale.reduce((total, { plan }) => total + plan.units.length, 0),
|
||||||
|
metadataTexts: stale.reduce((total, { plan }) => total + (plan.metadataText ? 1 : 0), 0),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -141,29 +151,41 @@ export const indexSpeechSources = async (
|
||||||
upToDate: batch.upToDate,
|
upToDate: batch.upToDate,
|
||||||
skippedNoText: batch.skippedNoText,
|
skippedNoText: batch.skippedNoText,
|
||||||
units: 0,
|
units: 0,
|
||||||
|
metadataTexts: 0,
|
||||||
promptTokens: 0,
|
promptTokens: 0,
|
||||||
}
|
}
|
||||||
if (batch.stale.length === 0) return { ...base, indexed: 0, failed: 0 }
|
if (batch.stale.length === 0) return { ...base, indexed: 0, failed: 0 }
|
||||||
|
|
||||||
const texts = batch.stale.flatMap(({ plan }) => plan.units.map((unit) => unit.text))
|
// C237 — the official metadata rides the same ordered provider call, right
|
||||||
|
// after the units of each speech; it feeds the speech vector only.
|
||||||
|
const texts = batch.stale.flatMap(({ plan }) => [
|
||||||
|
...plan.units.map((unit) => unit.text),
|
||||||
|
...(plan.metadataText ? [plan.metadataText] : []),
|
||||||
|
])
|
||||||
const embedded = await embedTexts(texts)
|
const embedded = await embedTexts(texts)
|
||||||
if (!embedded) {
|
if (!embedded) {
|
||||||
return { ...base, indexed: 0, failed: batch.stale.length }
|
return { ...base, indexed: 0, failed: batch.stale.length }
|
||||||
}
|
}
|
||||||
// The whole batch was embedded already: the tokens were spent even if a
|
// The whole batch was embedded already: the tokens were spent even if a
|
||||||
// later write fails.
|
// later write fails.
|
||||||
const spent = { units: texts.length, promptTokens: embedded.promptTokens }
|
const spent = {
|
||||||
|
units: batch.units,
|
||||||
|
metadataTexts: batch.metadataTexts,
|
||||||
|
promptTokens: embedded.promptTokens,
|
||||||
|
}
|
||||||
|
|
||||||
let cursor = 0
|
let cursor = 0
|
||||||
let indexed = 0
|
let indexed = 0
|
||||||
for (const { source, plan } of batch.stale) {
|
for (const { source, plan } of batch.stale) {
|
||||||
const vectors = embedded.vectors.slice(cursor, cursor + plan.units.length)
|
const vectors = embedded.vectors.slice(cursor, cursor + plan.units.length)
|
||||||
cursor += plan.units.length
|
cursor += plan.units.length
|
||||||
|
const metadataVector = plan.metadataText ? (embedded.vectors[cursor] ?? null) : null
|
||||||
|
if (plan.metadataText) cursor += 1
|
||||||
if (vectors.length !== plan.units.length) {
|
if (vectors.length !== plan.units.length) {
|
||||||
return { ...base, ...spent, indexed, failed: batch.stale.length - indexed }
|
return { ...base, ...spent, indexed, failed: batch.stale.length - indexed }
|
||||||
}
|
}
|
||||||
|
|
||||||
const speechVector = meanPoolVectors(vectors)
|
const speechVector = composeSpeechVector(vectors, metadataVector)
|
||||||
await withPayloadTransaction(payload, async ({ req }) => {
|
await withPayloadTransaction(payload, async ({ req }) => {
|
||||||
await payload.delete({
|
await payload.delete({
|
||||||
collection: 'speechEmbedding',
|
collection: 'speechEmbedding',
|
||||||
|
|
@ -299,6 +321,7 @@ export type SpeechIndexPreview = {
|
||||||
skippedNoText: number
|
skippedNoText: number
|
||||||
toIndex: number
|
toIndex: number
|
||||||
units: number
|
units: number
|
||||||
|
metadataTexts: number
|
||||||
}
|
}
|
||||||
|
|
||||||
export const previewSpeechIndex = (
|
export const previewSpeechIndex = (
|
||||||
|
|
@ -310,4 +333,5 @@ export const previewSpeechIndex = (
|
||||||
skippedNoText: batch.skippedNoText,
|
skippedNoText: batch.skippedNoText,
|
||||||
toIndex: batch.stale.length,
|
toIndex: batch.stale.length,
|
||||||
units: batch.units,
|
units: batch.units,
|
||||||
|
metadataTexts: batch.metadataTexts,
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -613,6 +613,70 @@ describe('speech acervo (C154)', () => {
|
||||||
).toContain(`Primeiro trecho ${runId}`)
|
).toContain(`Primeiro trecho ${runId}`)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('folds the official metadata into the speech vector (C237)', async () => {
|
||||||
|
const runId = randomUUID().slice(0, 8)
|
||||||
|
const offTopic = [0.5, Math.sqrt(3) / 2]
|
||||||
|
const summary = `Processo de impeachment da Presidenta Dilma Rousseff ${runId}`
|
||||||
|
const idWithMetadata = await createSpeech({
|
||||||
|
speechAt: '1988-01-01T10:00',
|
||||||
|
year: 1988,
|
||||||
|
summary,
|
||||||
|
keywords: ['IMPEACHMENT'],
|
||||||
|
segments: [{ startSeconds: 0, endSeconds: 3, text: `Trecho sem o tema ${runId}` }],
|
||||||
|
})
|
||||||
|
const idWithout = await createSpeech({
|
||||||
|
speechAt: '1988-01-02T10:00',
|
||||||
|
year: 1988,
|
||||||
|
segments: [{ startSeconds: 0, endSeconds: 3, text: `Trecho sem o tema ${runId}` }],
|
||||||
|
})
|
||||||
|
const embedCalls: string[][] = []
|
||||||
|
const embedTexts = async (texts: readonly string[]) => {
|
||||||
|
embedCalls.push([...texts])
|
||||||
|
return {
|
||||||
|
vectors: texts.map((text) =>
|
||||||
|
text.includes(runId) && text.startsWith('Processo') ? [1, 0] : offTopic,
|
||||||
|
),
|
||||||
|
promptTokens: 5,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await indexSpeechSources(
|
||||||
|
payload,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: idWithMetadata,
|
||||||
|
summary,
|
||||||
|
keywords: ['IMPEACHMENT'],
|
||||||
|
segments: [{ order: 1, startSeconds: 0, text: `Trecho sem o tema ${runId}` }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: idWithout,
|
||||||
|
segments: [{ order: 1, startSeconds: 0, text: `Trecho sem o tema ${runId}` }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
{ embedTexts },
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ indexed: 2, failed: 0, units: 2, metadataTexts: 1 })
|
||||||
|
expect(embedCalls[0]).toEqual([
|
||||||
|
`Trecho sem o tema ${runId}`,
|
||||||
|
`${summary}\nIMPEACHMENT`,
|
||||||
|
`Trecho sem o tema ${runId}`,
|
||||||
|
])
|
||||||
|
|
||||||
|
const { coordinator } = await createUsers()
|
||||||
|
const data = await loadSpeechAcervoPageData(
|
||||||
|
payload,
|
||||||
|
coordinator,
|
||||||
|
{ q: `tema ${runId}`, mode: 'tema', year: '1988' },
|
||||||
|
async () => [1, 0],
|
||||||
|
)
|
||||||
|
// Both passages never state the theme; the official metadata is what ranks
|
||||||
|
// the speech that carries it above the identical spoken-only neighbour.
|
||||||
|
expect(data.rows.map((row) => row.id)).toEqual([idWithMetadata, idWithout])
|
||||||
|
expect(data.rows[0]?.semanticMatch).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
it('cascades the index rows when the speech is deleted (C229)', async () => {
|
it('cascades the index rows when the speech is deleted (C229)', async () => {
|
||||||
const id = await createSpeech()
|
const id = await createSpeech()
|
||||||
await seedSpeechEmbedding(id, 'speech', null, [1, 0])
|
await seedSpeechEmbedding(id, 'speech', null, [1, 0])
|
||||||
|
|
|
||||||
|
|
@ -112,6 +112,27 @@ describe('formatSpeechIndexReport', () => {
|
||||||
expect(lines[2]).toContain('tokens: 123456')
|
expect(lines[2]).toContain('tokens: 123456')
|
||||||
expect(lines[2]).toContain('custo estimado: US$ 0.0012')
|
expect(lines[2]).toContain('custo estimado: US$ 0.0012')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('reports the metadata inputs on top of the stored units (C237)', () => {
|
||||||
|
const dryRun = formatSpeechIndexReport({ ...base, mode: 'dry-run', metadataTexts: 42 })
|
||||||
|
expect(dryRun[2]).toContain('trechos a vetorizar: 300 (+42 metadados)')
|
||||||
|
|
||||||
|
const write = formatSpeechIndexReport({
|
||||||
|
...base,
|
||||||
|
mode: 'index',
|
||||||
|
metadataTexts: 42,
|
||||||
|
indexed: 15,
|
||||||
|
failed: 0,
|
||||||
|
promptTokens: 10,
|
||||||
|
costUsd: 0,
|
||||||
|
})
|
||||||
|
expect(write[2]).toContain('trechos vetorizados: 300 (+42 metadados)')
|
||||||
|
|
||||||
|
// No metadata (speech without summary/title/keywords) keeps the old shape.
|
||||||
|
expect(formatSpeechIndexReport({ ...base, mode: 'dry-run' })[2]).toBe(
|
||||||
|
'trechos a vetorizar: 300',
|
||||||
|
)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('formatSpeechIndexProbe', () => {
|
describe('formatSpeechIndexProbe', () => {
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
import { describe, expect, it } from 'vitest'
|
import { describe, expect, it } from 'vitest'
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
composeSpeechVector,
|
||||||
cosineSimilarity,
|
cosineSimilarity,
|
||||||
dotProduct,
|
dotProduct,
|
||||||
meanPoolVectors,
|
meanPoolVectors,
|
||||||
|
|
@ -11,8 +12,10 @@ import {
|
||||||
planSpeechIndex,
|
planSpeechIndex,
|
||||||
rankSemanticHits,
|
rankSemanticHits,
|
||||||
sortSemanticHits,
|
sortSemanticHits,
|
||||||
|
SPEECH_METADATA_VECTOR_WEIGHT,
|
||||||
SPEECH_SEMANTIC_MIN_COSINE,
|
SPEECH_SEMANTIC_MIN_COSINE,
|
||||||
speechEmbeddingHash,
|
speechEmbeddingHash,
|
||||||
|
speechMetadataText,
|
||||||
speechSemanticSourceText,
|
speechSemanticSourceText,
|
||||||
speechTextWindows,
|
speechTextWindows,
|
||||||
type SpeechSemanticCandidate,
|
type SpeechSemanticCandidate,
|
||||||
|
|
@ -246,4 +249,65 @@ describe('planSpeechIndex', () => {
|
||||||
planSpeechIndex(speech, { model: 'b' })?.speechHash,
|
planSpeechIndex(speech, { model: 'b' })?.speechHash,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('plans the official metadata and pins it in the hash (C237)', () => {
|
||||||
|
const speech = {
|
||||||
|
summary: 'Processo de impeachment da Presidenta Dilma Rousseff.',
|
||||||
|
keywords: ['IMPEACHMENT', 'DILMA ROUSSEFF'],
|
||||||
|
segments: [{ order: 1, startSeconds: 0, text: 'Sr. Presidente...' }],
|
||||||
|
}
|
||||||
|
const plan = planSpeechIndex(speech, { model })
|
||||||
|
|
||||||
|
expect(plan?.metadataText).toBe(
|
||||||
|
'Processo de impeachment da Presidenta Dilma Rousseff.\nIMPEACHMENT, DILMA ROUSSEFF',
|
||||||
|
)
|
||||||
|
expect(plan?.speechHash).toBe(
|
||||||
|
speechEmbeddingHash(
|
||||||
|
'segment:1:Sr. Presidente...\nmetadata:Processo de impeachment da Presidenta Dilma Rousseff.\nIMPEACHMENT, DILMA ROUSSEFF',
|
||||||
|
model,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
// Editing the summary or the keywords must reindex the speech.
|
||||||
|
expect(
|
||||||
|
planSpeechIndex({ ...speech, summary: 'Outro sumário' }, { model })?.speechHash,
|
||||||
|
).not.toBe(plan?.speechHash)
|
||||||
|
expect(planSpeechIndex({ ...speech, keywords: ['OUTRA'] }, { model })?.speechHash).not.toBe(
|
||||||
|
plan?.speechHash,
|
||||||
|
)
|
||||||
|
// A speech without metadata keeps the pre-C237 hash bytes.
|
||||||
|
expect(planSpeechIndex({ segments: speech.segments }, { model })?.metadataText).toBeNull()
|
||||||
|
expect(planSpeechIndex({ segments: speech.segments }, { model })?.speechHash).toBe(
|
||||||
|
speechEmbeddingHash('segment:1:Sr. Presidente...', model),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('speechMetadataText (C237)', () => {
|
||||||
|
it('prefers the summary over the title and joins the raw keywords', () => {
|
||||||
|
expect(
|
||||||
|
speechMetadataText({ summary: 'Resumo', title: 'Título', keywords: ['SUS', ' Saúde '] }),
|
||||||
|
).toBe('Resumo\nSUS, Saúde')
|
||||||
|
expect(speechMetadataText({ title: 'Título', keywords: [] })).toBe('Título')
|
||||||
|
expect(speechMetadataText({ summary: ' ', keywords: [' '] })).toBeNull()
|
||||||
|
expect(speechMetadataText({})).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('composeSpeechVector (C237)', () => {
|
||||||
|
const unit = [0.6, 0.8]
|
||||||
|
|
||||||
|
it('keeps the pure mean without metadata', () => {
|
||||||
|
expect(composeSpeechVector([unit], null)).toEqual(meanPoolVectors([unit]))
|
||||||
|
// A metadata vector of another dimension never poisons the mean.
|
||||||
|
expect(composeSpeechVector([unit], [1, 0, 0])).toEqual(meanPoolVectors([unit]))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('pulls the speech vector toward the weighted metadata', () => {
|
||||||
|
const composed = composeSpeechVector([unit], [1, 0])
|
||||||
|
|
||||||
|
expect(composed).toEqual(
|
||||||
|
normalizeVector([unit[0] + SPEECH_METADATA_VECTOR_WEIGHT * 1, unit[1]]),
|
||||||
|
)
|
||||||
|
expect(cosineSimilarity(composed, [1, 0])).toBeGreaterThan(cosineSimilarity(unit, [1, 0]))
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue