teqo-backup/scripts/seed-tse-results.mjs

/**
 * Seeds electionTally / electionCandidateVote / electionCandidate from TSE
 * open data (Bahia scope). Default year: 2022 (full ticket). Historical years
 * 2014/2018 import deputado_federal + presidente + governador turno 1 (E2
 * federal trend series + E8 majoritarian baseline for teto do campo/roll-off
 * validation across cycles). Skips deputado_estadual for 2014/2018 — no
 * consumer needs the state-assembly race historically. The country-wide
 * (non-BA) files are always loaded too — see the inline comment above
 * `voteBa`/`voteBr` below for why (presidente is BR-scope in TSE's data).
 *
 * Provenance (do not strip):
 * - Portal 2022: https://dadosabertos.tse.jus.br/dataset/resultados-2022
 * - Portal 2018: https://dadosabertos.tse.jus.br/dataset/resultados-2018
 * - Portal 2014: https://dadosabertos.tse.jus.br/dataset/resultados-2014
 * - Portal candidatos: https://dadosabertos.tse.jus.br/dataset/candidatos-{year}
 * - CDN zips: https://cdn.tse.jus.br/estatistica/sead/odsele/{family}/{family}_{year}.zip
 * - License: Creative Commons Atribuição (dados abertos TSE)
 * - electionCandidateVote.votes ← QT_VOTOS_NOMINAIS (votos apurados; validade no tally)
 * - v1: only voteType=nominal (no votacao_partido_munzona / legenda rows)
 *
 * Pinned SHA-256 (update after verifying a fresh download):
 * - 2014/2018: log-only until first verified import in CI/local
 * - 2022: log-only (existing deployments may have cached zips)
 *
 * Safety: refuses non-local DATABASE_URL unless ALLOW_REMOTE_DB=true.
 * Idempotent: replace-by-scope (year, office, turn) via drizzle transaction.
 *
 * Usage:
 *   pnpm db:seed:tse
 *   pnpm db:seed:tse -- --year=2018
 *   pnpm db:seed:tse -- --year=2014
 *   ALLOW_REMOTE_DB=true pnpm db:seed:tse
 *   TSE_CACHE_DIR=./data/tse pnpm db:seed:tse
 */

import { join } from 'node:path'
import { getPayload } from 'payload'
import { dieWithLabel, ensureCachedDownload, loadCliEnv } from './lib/cli.mjs'

import { assertLocalDatabase } from './assert-local-database.mjs'

loadCliEnv()

const config = (await import('../src/payload.config.ts')).default
const { parseTseCsvBuffer } = await import('../src/lib/electionResultsCsv.ts')
const { buildElectionResultsFromCsvRows, HISTORICAL_BASELINE_OFFICES } =
  await import('../src/lib/electionResultsBuild.ts')
const { downloadToBuffer, readZipEntry } = await import('../src/lib/electionResultsZip.ts')
const { buildImportBundles, importElectionBundles } =
  await import('../src/utilities/electionResultsImport.ts')

const SUPPORTED_YEARS = [2014, 2018, 2022]

/** Optional integrity pins — null = warn on mismatch only after download. */
const EXPECTED_SHA256 = {
  2014: null,
  2018: null,
  2022: null,
}

const sourcesForYear = (year) => [
  {
    key: `votacao_candidato_munzona_${year}`,
    url: `https://cdn.tse.jus.br/estatistica/sead/odsele/votacao_candidato_munzona/votacao_candidato_munzona_${year}.zip`,
    expectedSha256: EXPECTED_SHA256[year],
  },
  {
    key: `detalhe_votacao_munzona_${year}`,
    url: `https://cdn.tse.jus.br/estatistica/sead/odsele/detalhe_votacao_munzona/detalhe_votacao_munzona_${year}.zip`,
    expectedSha256: EXPECTED_SHA256[year],
  },
  {
    key: `consulta_cand_${year}`,
    url: `https://cdn.tse.jus.br/estatistica/sead/odsele/consulta_cand/consulta_cand_${year}.zip`,
    expectedSha256: EXPECTED_SHA256[year],
  },
]

const csvNamesForYear = (year) => ({
  voteBa: `votacao_candidato_munzona_${year}_BA.csv`,
  voteBr: `votacao_candidato_munzona_${year}_BR.csv`,
  detalheBa: `detalhe_votacao_munzona_${year}_BA.csv`,
  detalheBr: `detalhe_votacao_munzona_${year}_BR.csv`,
  candBa: `consulta_cand_${year}_BA.csv`,
  candBr: `consulta_cand_${year}_BR.csv`,
})

const die = dieWithLabel('seed:tse')

const parseYearArg = () => {
  const explicit = process.argv.find((arg) => arg.startsWith('--year='))
  if (!explicit) return 2022
  const year = Number(explicit.slice('--year='.length))
  if (!SUPPORTED_YEARS.includes(year)) {
    die(`Unsupported --year=${year}. Supported: ${SUPPORTED_YEARS.join(', ')}`)
  }
  return year
}

const cacheDir = () => process.env.TSE_CACHE_DIR || join(process.cwd(), 'data', 'tse')

const ensureCachedZip = ({ key, url, expectedSha256 }) =>
  ensureCachedDownload({
    label: 'seed:tse',
    key,
    url,
    ext: 'zip',
    cacheDir: cacheDir(),
    expectedSha256,
    download: downloadToBuffer,
  })

const loadCsvFromZip = async (zipBuffer, fileName) => {
  const entry = await readZipEntry(zipBuffer, (name) => name.endsWith(fileName))
  if (!entry) die(`ZIP entry not found: ${fileName}`)
  console.log(`[seed:tse] parsing ${entry.fileName} (${entry.buffer.length} bytes)`)
  return parseTseCsvBuffer(entry.buffer)
}

const main = async () => {
  const year = parseYearArg()
  const historicalOnly = year === 2014 || year === 2018

  assertLocalDatabase(
    'seed:tse',
    'This seed WRITES election reference data and is meant for the local Postgres.\n' +
      '  1. pnpm db:start\n' +
      '  2. set DATABASE_URL=postgresql://teqo:teqo@localhost:5432/teqo in .env.local',
  )

  const sources = sourcesForYear(year)
  const [votacaoZip, detalheZip, candZip] = await Promise.all(sources.map(ensureCachedZip))

  console.log(
    `[seed:tse] year=${year} scope=${historicalOnly ? 'deputado_federal + presidente + governador T1 BA' : 'full ticket BA'}`,
  )
  console.log('[seed:tse] provenance:')
  for (const source of [votacaoZip, detalheZip, candZip]) {
    console.log(`  - ${source.url}`)
    console.log(`    sha256=${source.hash}`)
  }

  const csv = csvNamesForYear(year)

  // Sequential: BA vote CSV alone is hundreds of MB; parallel parse spikes peak RAM.
  // TSE splits "_BA.csv" by unidade eleitoral (UE), not by voter's state: presidente
  // runs under UE=BR, so its per-município/zona votes+tallies (BA municípios
  // included) live only in the "_BR.csv" files, never in "_BA.csv" — those
  // files are always needed once presidente is in scope, historical or not.
  // candBr registers presidente candidates (also a national/BR-scope race).
  const voteBa = await loadCsvFromZip(votacaoZip.buffer, csv.voteBa)
  const voteBr = await loadCsvFromZip(votacaoZip.buffer, csv.voteBr)
  const detalheBa = await loadCsvFromZip(detalheZip.buffer, csv.detalheBa)
  const detalheBr = await loadCsvFromZip(detalheZip.buffer, csv.detalheBr)
  const candBa = await loadCsvFromZip(candZip.buffer, csv.candBa)
  const candBr = await loadCsvFromZip(candZip.buffer, csv.candBr)

  const built = buildElectionResultsFromCsvRows({
    voteRows: [...voteBa, ...voteBr],
    tallyRows: [...detalheBa, ...detalheBr],
    candBaRows: candBa,
    candBrRows: candBr,
    year,
    ...(historicalOnly ? { offices: HISTORICAL_BASELINE_OFFICES } : {}),
  })

  if (built.unknownMunicipalities.length > 0) {
    die(
      `Municípios TSE sem mapeamento canônico (${built.unknownMunicipalities.length}):\n  - ` +
        built.unknownMunicipalities.slice(0, 20).join('\n  - ') +
        (built.unknownMunicipalities.length > 20 ? '\n  - …' : ''),
    )
  }

  console.log(
    `[seed:tse] parsed votes=${built.votes.length} tallies=${built.tallies.length} candidates=${built.candidates.length}`,
  )

  const bundles = buildImportBundles(built)
  console.log(`[seed:tse] importing ${bundles.length} scopes…`)

  const payload = await getPayload({ config })
  const counts = await importElectionBundles(payload, bundles)

  let votes = 0
  let tallies = 0
  let candidates = 0
  for (const count of counts) {
    votes += count.votesInserted
    tallies += count.talliesInserted
    candidates += count.candidatesInserted
    console.log(
      `  ${count.scope.office} turn=${count.scope.turn}: votes=${count.votesInserted} tallies=${count.talliesInserted} candidates=${count.candidatesInserted}`,
    )
  }

  console.log(
    `\n[seed:tse] done (year=${year}). inserted votes=${votes} tallies=${tallies} candidates=${candidates}`,
  )
  process.exit(0)
}

main().catch((error) => {
  console.error('[seed:tse] fatal:', error)
  process.exit(1)
})