teqo-backup/scripts/lib/camaraFetch.mjs

/**
 * HTTP surface shared by the Câmara speech scripts (C152 pilot + C153 import):
 * the open-data endpoints, the anti-crawl browser UA, bounded retry, the async
 * VOD poll and the Deep Infra transcription call. Kept apart from
 * `camaraSpeeches.mjs` (pure) so the fragile rules stay unit-testable without
 * network.
 */
import {
  buildVodUrl,
  CAMARA_USER_AGENT,
  DEEPINFRA_TRANSCRIBE_URL,
  DEEPINFRA_WHISPER_MODEL,
  normalizeTranscription,
  parseVodStatus,
} from './camaraSpeeches.mjs'

const camaraHeaders = { 'User-Agent': CAMARA_USER_AGENT, Accept: 'application/json' }

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))

async function fetchOnce(url, { timeoutMs = 30_000, headers = camaraHeaders } = {}) {
  const response = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
  if (!response.ok) throw new Error(`HTTP ${response.status} em ${url}`)
  return response
}

/** The Câmara API drops connections now and then — bounded retry for GETs. */
export async function getText(url, { attempts = 3, ...options } = {}) {
  let lastError
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    try {
      return await (await fetchOnce(url, options)).text()
    } catch (error) {
      lastError = error
      if (attempt < attempts) await sleep(2_000)
    }
  }
  throw lastError
}

export const getJson = async (url, options) => JSON.parse(await getText(url, options))

/**
 * Pagination-grade GET: the open-data API throws transient 500s and hangs on
 * deep pages (seen 2026-09-13 on the 55ª page 4), and `getText`'s own 3
 * attempts are too impatient for a 1.011-speech backfill. Bounded backoff;
 * the caller owns the failure policy (the import aborts only the legislature).
 *
 * @param {string} url
 * @param {{ attempts?: number, timeoutMs?: number, label?: string }} [options]
 */
export async function getJsonWithBackoff(
  url,
  { attempts = 5, timeoutMs = 45_000, label = 'camara' } = {},
) {
  let lastError
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    try {
      return JSON.parse(await getText(url, { attempts: 1, timeoutMs }))
    } catch (error) {
      lastError = error
      if (attempt < attempts) {
        const waitMs = Math.min(30_000, 2 ** attempt * 1_000)
        console.log(
          `[${label}] ${error?.message ?? error} — retry ${attempt}/${attempts - 1} em ${waitMs / 1000}s`,
        )
        await sleep(waitMs)
      }
    }
  }
  throw lastError
}

export const downloadToBuffer = async (url, { attempts = 3, timeoutMs = 120_000 } = {}) => {
  let lastError
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    try {
      const response = await fetchOnce(url, {
        timeoutMs,
        headers: { 'User-Agent': CAMARA_USER_AGENT },
      })
      return Buffer.from(await response.arrayBuffer())
    } catch (error) {
      lastError = error
      if (attempt < attempts) await sleep(3_000)
    }
  }
  throw lastError
}

export const speechesUrl = (deputyId, from, to) =>
  `https://dadosabertos.camara.leg.br/api/v2/deputados/${deputyId}/discursos` +
  `?dataInicio=${from}&dataFim=${to}&itens=100&ordenarPor=dataHoraInicio&ordem=ASC`

export const eventsUrl = (date) =>
  `https://dadosabertos.camara.leg.br/api/v2/eventos?dataInicio=${date}&dataFim=${date}&itens=100`

export const eventPageUrl = (eventId) => `https://www.camara.leg.br/evento-legislativo/${eventId}`

const VOD_POLL_INTERVAL_MS = 5_000
const VOD_POLL_MAX_ATTEMPTS = 40

/** Polls the async VOD generation until PRONTO (or gives up). */
export async function resolveVod(eventId, excerpt) {
  const url = buildVodUrl(eventId, excerpt.audioId, excerpt.tMs)
  let lastStatus = null
  for (let attempt = 1; attempt <= VOD_POLL_MAX_ATTEMPTS; attempt += 1) {
    const status = parseVodStatus(await getJson(url, { timeoutMs: 60_000 }))
    lastStatus = status
    if (status.state === 'PRONTO' || status.state === 'INDISPONIVEL') {
      return { url, status, attempts: attempt }
    }
    await sleep(VOD_POLL_INTERVAL_MS)
  }
  return { url, status: lastStatus, attempts: VOD_POLL_MAX_ATTEMPTS }
}

/**
 * Lightweight reachability probe of one stored VOD link (C155 sample check):
 * GET with a 1 KiB `Range` so the MP4 body never streams, falling back to HEAD
 * when the server refuses the ranged GET (405/501). Never throws — a network
 * error becomes an `ok: false` result so one dead link cannot abort the report.
 *
 * @param {string | null | undefined} url
 * @returns {Promise<{ url: string | null, status: number | null, ok: boolean, note: string }>}
 */
export async function probeVodLink(url, { timeoutMs = 20_000 } = {}) {
  if (!url) return { url: null, status: null, ok: false, note: 'sem link' }

  const request = async (method) => {
    const response = await fetch(url, {
      method,
      headers: {
        'User-Agent': CAMARA_USER_AGENT,
        ...(method === 'GET' ? { Range: 'bytes=0-1023' } : {}),
      },
      signal: AbortSignal.timeout(timeoutMs),
    })
    const contentType = response.headers.get('content-type')
    const note = `HTTP ${response.status}${contentType ? ` ${contentType}` : ''}`
    await response.body?.cancel().catch(() => undefined)
    return { url, status: response.status, ok: response.ok, note }
  }

  try {
    const first = await request('GET')
    if (first.status === 405 || first.status === 501) return await request('HEAD')
    return first
  } catch (error) {
    return { url, status: null, ok: false, note: error?.message ?? String(error) }
  }
}

export const DEEPINFRA_COST_PER_MINUTE_USD = 0.00045

/**
 * Sends the excerpt MP4 (video/mp4 is accepted; no ffmpeg) to Deep Infra's
 * OpenAI-compatible endpoint and returns the normalized transcription.
 * Throws on missing key or provider error — the caller owns the failure policy.
 */
export async function transcribeSpeechAudio(buffer) {
  const apiKey = process.env.DEEPINFRA_API_KEY
  if (!apiKey) {
    throw new Error('DEEPINFRA_API_KEY ausente — configure no .env.local ou no env do processo.')
  }
  const form = new FormData()
  form.append('file', new Blob([buffer], { type: 'video/mp4' }), 'trecho.mp4')
  form.append('model', DEEPINFRA_WHISPER_MODEL)
  form.append('language', 'pt')
  form.append('response_format', 'verbose_json')
  form.append('timestamp_granularities[]', 'segment')
  const startedAt = Date.now()
  const response = await fetch(DEEPINFRA_TRANSCRIBE_URL, {
    method: 'POST',
    headers: { Authorization: `Bearer ${apiKey}` },
    body: form,
    signal: AbortSignal.timeout(600_000),
  })
  if (!response.ok) {
    throw new Error(`Deep Infra HTTP ${response.status}: ${(await response.text()).slice(0, 300)}`)
  }
  return {
    transcription: normalizeTranscription(await response.json()),
    elapsedMs: Date.now() - startedAt,
  }
}