diff --git a/scripts/import-camara-speeches.mjs b/scripts/import-camara-speeches.mjs index 02b750ac..7864a492 100644 --- a/scripts/import-camara-speeches.mjs +++ b/scripts/import-camara-speeches.mjs @@ -47,6 +47,7 @@ import { LEGISLATURE_RANGES, SOLLA_DEPUTY_ID, aggregateBackfillRuns, + dateRangeChunks, legislatureForDate, matchExcerpt, parseDurationToSeconds, @@ -227,7 +228,7 @@ async function loadEventPage(eventId, options) { return html } -async function fetchSpeeches(deputyId, from, to) { +async function fetchSpeechesRange(deputyId, from, to) { const speeches = [] for (let page = 1; ; page += 1) { const body = await getJsonWithBackoff(`${speechesUrl(deputyId, from, to)}&pagina=${page}`, { @@ -242,6 +243,32 @@ async function fetchSpeeches(deputyId, from, to) { return speeches } +/** + * Lista a faixa inteira; se a API recusar uma página funda (500/hang + * persistente, visto na 55ª em 2026-09-13), cai para a listagem por ano — cada + * chunk é raso e um ano quebrado não derruba os demais. Os chunks que falharem + * são reportados via `onWarning` (o run os registra como falha de `list`). + */ +async function fetchSpeeches(deputyId, from, to, onWarning = () => {}) { + try { + return await fetchSpeechesRange(deputyId, from, to) + } catch (error) { + onWarning(`listagem ${from}..${to} falhou (${error?.message}); tentando por ano`) + const speeches = [] + let listedChunks = 0 + for (const [chunkFrom, chunkTo] of dateRangeChunks(from, to)) { + try { + speeches.push(...(await fetchSpeechesRange(deputyId, chunkFrom, chunkTo))) + listedChunks += 1 + } catch (chunkError) { + onWarning(`listagem ${chunkFrom}..${chunkTo} falhou: ${chunkError?.message}`) + } + } + if (listedChunks === 0) throw error + return speeches + } +} + // --------------------------------------------------------------------------- // Pipeline // --------------------------------------------------------------------------- @@ -638,7 +665,9 @@ const runLegislature = async (payload, options, { legislature, from, to, label } const startedAt = Date.now() console.log(`\n[camara:import] === ${label} (${from}..${to}) ===`) - const speeches = await fetchSpeeches(SOLLA_DEPUTY_ID, from, to) + const speeches = await fetchSpeeches(SOLLA_DEPUTY_ID, from, to, (message) => { + run.failures.push({ sourceKey: null, speechAt: null, stage: 'list', message }) + }) const targets = options.limit === null ? speeches : speeches.slice(0, options.limit) console.log(`[camara:import] ${speeches.length} discursos na API; processando ${targets.length}`) if (targets.length === 0) throw new Error(`nenhum discurso em ${from}..${to}.`) diff --git a/scripts/lib/camaraSpeeches.mjs b/scripts/lib/camaraSpeeches.mjs index 65580c4f..01698860 100644 --- a/scripts/lib/camaraSpeeches.mjs +++ b/scripts/lib/camaraSpeeches.mjs @@ -399,6 +399,33 @@ export const normalizeTranscription = (json) => { } } +/** + * Splits an inclusive `YYYY-MM-DD` range into calendar-year chunks (no + * overlap; first/last partial). The backfill fallback uses it when the API + * refuses a deep page: a shallow per-year listing still recovers the years + * the broken offset does not cover. + * + * @param {string} from + * @param {string} to + * @returns {Array<[string, string]>} + */ +export const dateRangeChunks = (from, to) => { + const chunks = [] + if (!/^\d{4}-\d{2}-\d{2}$/.test(String(from)) || !/^\d{4}-\d{2}-\d{2}$/.test(String(to))) { + return chunks + } + let start = String(from) + while (start <= String(to)) { + const yearEnd = `${start.slice(0, 4)}-12-31` + const end = yearEnd < String(to) ? yearEnd : String(to) + chunks.push([start, end]) + const next = new Date(`${end}T00:00:00Z`) + next.setUTCDate(next.getUTCDate() + 1) + start = next.toISOString().slice(0, 10) + } + return chunks +} + /** Numeric blocks of one run that `aggregateBackfillRuns` folds together. */ const BACKFILL_TOTAL_KEYS = [ 'listed', diff --git a/tests/unit/camaraSpeeches.unit.spec.ts b/tests/unit/camaraSpeeches.unit.spec.ts index 8837e70c..b878d3c7 100644 --- a/tests/unit/camaraSpeeches.unit.spec.ts +++ b/tests/unit/camaraSpeeches.unit.spec.ts @@ -6,6 +6,7 @@ import { aggregateBackfillRuns, buildVodUrl, clockToSeconds, + dateRangeChunks, legislatureForDate, matchExcerpt, normalizeSpeakerName, @@ -634,3 +635,25 @@ describe('selectLinkSample', () => { ).toEqual([]) }) }) + +describe('dateRangeChunks', () => { + it('splits a multi-year range into non-overlapping calendar years', () => { + expect(dateRangeChunks('2015-02-01', '2019-01-31')).toEqual([ + ['2015-02-01', '2015-12-31'], + ['2016-01-01', '2016-12-31'], + ['2017-01-01', '2017-12-31'], + ['2018-01-01', '2018-12-31'], + ['2019-01-01', '2019-01-31'], + ]) + }) + + it('handles a single day and an exact year without spilling over', () => { + expect(dateRangeChunks('2023-02-07', '2023-02-07')).toEqual([['2023-02-07', '2023-02-07']]) + expect(dateRangeChunks('2023-01-01', '2023-12-31')).toEqual([['2023-01-01', '2023-12-31']]) + }) + + it('returns no chunks for malformed dates', () => { + expect(dateRangeChunks('2023-1-1', '2023-12-31')).toEqual([]) + expect(dateRangeChunks('', '2023-12-31')).toEqual([]) + }) +})