mirror of
https://github.com/fsolla/teqo.git
synced 2026-11-16 20:13:04 +00:00
319 lines
11 KiB
TypeScript
319 lines
11 KiB
TypeScript
import { z } from 'zod'
|
|
|
|
/**
|
|
* C215 — pure contract of the web-speech ingestion: the platform taxonomy, the
|
|
* finding-batch schema the discovery (C218) hands over, the natural key
|
|
* (`web:<platform>:<externalId|url>`) and the canonical URL normalization that
|
|
* keeps re-runs idempotent. No I/O: the CLI, the pipeline and the tests share
|
|
* this module.
|
|
*/
|
|
|
|
export const WEB_SPEECH_PLATFORMS = [
|
|
{ value: 'youtube', label: 'YouTube' },
|
|
{ value: 'instagram', label: 'Instagram' },
|
|
{ value: 'radio', label: 'Rádio' },
|
|
{ value: 'audio', label: 'Áudio' },
|
|
] as const
|
|
|
|
export type WebSpeechPlatform = (typeof WEB_SPEECH_PLATFORMS)[number]['value']
|
|
|
|
const WEB_SPEECH_PLATFORM_VALUES = WEB_SPEECH_PLATFORMS.map((platform) => platform.value) as [
|
|
WebSpeechPlatform,
|
|
...WebSpeechPlatform[],
|
|
]
|
|
|
|
/** The honest label of a row without a known platform (C216 list/detail pill). */
|
|
const WEB_SPEECH_PLATFORM_UNKNOWN_LABEL = 'Outra plataforma'
|
|
|
|
export const webSpeechPlatformLabel = (value: WebSpeechPlatform | null | undefined): string => {
|
|
if (!value) return WEB_SPEECH_PLATFORM_UNKNOWN_LABEL
|
|
return WEB_SPEECH_PLATFORMS.find((platform) => platform.value === value)?.label ?? value
|
|
}
|
|
|
|
/** Owner of the private upload collection slug (collection, config and S3). */
|
|
export const INTERNET_SPEECH_MEDIA_SLUG = 'internetSpeechMedia' as const
|
|
|
|
/**
|
|
* Platforms whose acquisition goes through yt-dlp. The others (`radio`,
|
|
* `audio`) are direct files: the finding must carry `mediaUrl` or the item
|
|
* fails honestly in the report — the script never guesses a page's media.
|
|
*/
|
|
const YTDLP_PLATFORMS: readonly WebSpeechPlatform[] = ['youtube', 'instagram']
|
|
|
|
export const requiresDirectMediaUrl = (platform: WebSpeechPlatform): boolean =>
|
|
!YTDLP_PLATFORMS.includes(platform)
|
|
|
|
const TRACKING_PARAMS = new Set(['fbclid', 'gclid', 'igshid', 'igsh', 'si', 'feature'])
|
|
|
|
const isTrackingParam = (name: string): boolean =>
|
|
TRACKING_PARAMS.has(name) || name.toLowerCase().startsWith('utm_')
|
|
|
|
const stripTrackingParams = (url: URL): void => {
|
|
for (const name of [...url.searchParams.keys()]) {
|
|
if (isTrackingParam(name)) url.searchParams.delete(name)
|
|
}
|
|
}
|
|
|
|
const withoutTrailingSlash = (pathname: string): string =>
|
|
pathname.length > 1 && pathname.endsWith('/') ? pathname.replace(/\/+$/, '') : pathname
|
|
|
|
const YOUTUBE_HOSTS = new Set(['youtube.com', 'm.youtube.com', 'music.youtube.com'])
|
|
const YOUTU_BE_HOSTS = new Set(['youtu.be'])
|
|
|
|
const normalizedHost = (url: URL): string => url.hostname.toLowerCase().replace(/^www\./, '')
|
|
|
|
const firstPathSegment = (pathname: string): string | null =>
|
|
pathname.split('/').find((segment) => segment !== '') ?? null
|
|
|
|
/** Video id from any YouTube URL shape (`watch`, `shorts`, `live`, `embed`, `youtu.be`). */
|
|
const youtubeVideoId = (url: URL): string | null => {
|
|
const host = normalizedHost(url)
|
|
if (YOUTU_BE_HOSTS.has(host)) return firstPathSegment(url.pathname)
|
|
if (!YOUTUBE_HOSTS.has(host)) return null
|
|
if (url.pathname === '/watch') return url.searchParams.get('v')
|
|
const match = /^\/(?:shorts|live|embed)\/([^/?#]+)/.exec(url.pathname)
|
|
return match?.[1] ?? null
|
|
}
|
|
|
|
/** Shortcode + kind (`reel`/`p`/`tv`) from any Instagram post URL. */
|
|
const instagramPost = (url: URL): { kind: string; code: string } | null => {
|
|
if (normalizedHost(url) !== 'instagram.com') return null
|
|
const match = /^\/(p|reel|reels|tv)\/([^/?#]+)/.exec(url.pathname)
|
|
if (!match) return null
|
|
return { kind: match[1] === 'reels' ? 'reel' : match[1], code: match[2] }
|
|
}
|
|
|
|
const parseUrl = (raw: string): URL | null => {
|
|
try {
|
|
return new URL(raw.trim())
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stable URL for identity and provenance: YouTube/Instagram collapse to the
|
|
* canonical post URL and every platform drops tracking params, the fragment
|
|
* and a trailing slash. An unparseable URL returns trimmed as-is (the finding
|
|
* schema validates the shape before this runs).
|
|
*/
|
|
export const canonicalWebSpeechUrl = (platform: WebSpeechPlatform, rawUrl: string): string => {
|
|
const url = parseUrl(rawUrl)
|
|
if (!url) return rawUrl.trim()
|
|
|
|
if (platform === 'youtube') {
|
|
const videoId = youtubeVideoId(url)
|
|
if (videoId) return `https://www.youtube.com/watch?v=${videoId}`
|
|
}
|
|
if (platform === 'instagram') {
|
|
const post = instagramPost(url)
|
|
if (post) return `https://www.instagram.com/${post.kind}/${post.code}/`
|
|
}
|
|
|
|
url.hash = ''
|
|
stripTrackingParams(url)
|
|
url.hostname = url.hostname.toLowerCase()
|
|
url.pathname = withoutTrailingSlash(url.pathname)
|
|
return url.toString()
|
|
}
|
|
|
|
/**
|
|
* Natural key of one web speech. `externalId` wins when the discovery already
|
|
* knows the platform id; otherwise the canonical URL is the identity — the
|
|
* same URL always produces the same key, so re-running never duplicates.
|
|
*/
|
|
export const webSpeechSourceKey = ({
|
|
platform,
|
|
url,
|
|
externalId,
|
|
}: {
|
|
platform: WebSpeechPlatform
|
|
url: string
|
|
externalId?: string | null
|
|
}): string => {
|
|
const id = externalId?.trim()
|
|
return id ? `web:${platform}:${id}` : `web:${platform}:${canonicalWebSpeechUrl(platform, url)}`
|
|
}
|
|
|
|
/** A web speech row is complete when it has both the transcript and the mirrored file. */
|
|
export const isCompleteWebSpeechState = (
|
|
state: { segmentCount: number; mirroredMedia: number | null } | null,
|
|
): boolean => state !== null && state.segmentCount > 0 && state.mirroredMedia !== null
|
|
|
|
const DOWNLOAD_FILENAME_MAX_LENGTH = 120
|
|
|
|
/**
|
|
* C216 (closes the C215 S4 defer) — the legible download name of a mirrored
|
|
* file: the speech title keeps the stored extension, control characters and
|
|
* path separators become spaces, dots/dashes stop trailing and the length is
|
|
* capped. Returns null when the stored file has no name or the title leaves
|
|
* nothing readable, so the caller falls back to the stored filename. The final
|
|
* `Content-Disposition` sanitization belongs to `privateMediaContentDisposition`.
|
|
*/
|
|
export const webSpeechDownloadFilename = ({
|
|
title,
|
|
storedFilename,
|
|
}: {
|
|
title?: string | null
|
|
storedFilename?: string | null
|
|
}): string | null => {
|
|
const stored = storedFilename?.trim()
|
|
if (!stored) return null
|
|
|
|
const lastDot = stored.lastIndexOf('.')
|
|
const extension = lastDot > 0 ? stored.slice(lastDot).toLowerCase() : ''
|
|
const base = (title ?? '')
|
|
.normalize('NFC')
|
|
.replace(/[\p{Cc}"\\/:*?<>|]+/gu, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
if (!base) return null
|
|
|
|
const maxBaseLength = Math.max(1, DOWNLOAD_FILENAME_MAX_LENGTH - extension.length)
|
|
const capped = base.slice(0, maxBaseLength).replace(/[.\s]+$/, '')
|
|
return capped ? `${capped}${extension}` : null
|
|
}
|
|
|
|
const PUBLISHED_AT_PATTERN = /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2}))?$/
|
|
const TIMEZONE_SUFFIX = /(?:Z|[+-]\d{2}:?\d{2})$/
|
|
|
|
const isValidCalendarDay = (year: string, month: string, day: string): boolean => {
|
|
const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day)))
|
|
return (
|
|
date.getUTCFullYear() === Number(year) &&
|
|
date.getUTCMonth() === Number(month) - 1 &&
|
|
date.getUTCDate() === Number(day)
|
|
)
|
|
}
|
|
|
|
/**
|
|
* `speechAt` (Brasília wall-clock text, same contract as the Câmara rows) from
|
|
* the discovery's `publishedAt` — day-only becomes midnight and an explicit
|
|
* timezone is converted to America/Bahia. Unrecognized values return null so
|
|
* the finding fails validation instead of inventing a date.
|
|
*/
|
|
export const webSpeechAtFromPublishedAt = (publishedAt: string): string | null => {
|
|
const trimmed = publishedAt.trim()
|
|
|
|
if (TIMEZONE_SUFFIX.test(trimmed)) {
|
|
const instant = new Date(trimmed)
|
|
if (Number.isNaN(instant.getTime())) return null
|
|
// sv-SE renders the ISO-like `YYYY-MM-DD HH:mm` for the requested zone.
|
|
return new Intl.DateTimeFormat('sv-SE', {
|
|
timeZone: 'America/Bahia',
|
|
year: 'numeric',
|
|
month: '2-digit',
|
|
day: '2-digit',
|
|
hour: '2-digit',
|
|
minute: '2-digit',
|
|
hour12: false,
|
|
})
|
|
.format(instant)
|
|
.replace(' ', 'T')
|
|
}
|
|
|
|
const match = PUBLISHED_AT_PATTERN.exec(trimmed)
|
|
if (!match) return null
|
|
const [, year, month, day, hour = '00', minute = '00'] = match
|
|
if (!isValidCalendarDay(year, month, day)) return null
|
|
if (Number(hour) > 23 || Number(minute) > 59) return null
|
|
return `${year}-${month}-${day}T${hour}:${minute}`
|
|
}
|
|
|
|
const MAX_TITLE_LENGTH = 300
|
|
const MAX_CHANNEL_LENGTH = 200
|
|
|
|
const webSpeechFindingSchema = z
|
|
.object({
|
|
platform: z.enum(WEB_SPEECH_PLATFORM_VALUES, { error: 'plataforma desconhecida' }),
|
|
url: z.string({ error: 'url obrigatória' }).trim().url('url inválida'),
|
|
mediaUrl: z.string().trim().url('mediaUrl inválida').optional(),
|
|
externalId: z.string().trim().min(1, 'externalId vazio').optional(),
|
|
title: z
|
|
.string()
|
|
.trim()
|
|
.min(1, 'título vazio')
|
|
.max(MAX_TITLE_LENGTH, `título até ${MAX_TITLE_LENGTH} caracteres`)
|
|
.optional(),
|
|
channel: z
|
|
.string()
|
|
.trim()
|
|
.min(1, 'canal vazio')
|
|
.max(MAX_CHANNEL_LENGTH, `canal até ${MAX_CHANNEL_LENGTH} caracteres`)
|
|
.optional(),
|
|
publishedAt: z
|
|
.string({ error: 'publishedAt obrigatório' })
|
|
.trim()
|
|
.min(4, 'publishedAt inválido'),
|
|
durationSeconds: z
|
|
.number({ error: 'durationSeconds inválido' })
|
|
.finite()
|
|
.nonnegative('durationSeconds inválido')
|
|
.optional(),
|
|
thumbnailUrl: z.string().trim().url('thumbnailUrl inválida').optional(),
|
|
})
|
|
.superRefine((finding, context) => {
|
|
if (requiresDirectMediaUrl(finding.platform) && !finding.mediaUrl) {
|
|
context.addIssue({
|
|
code: 'custom',
|
|
path: ['mediaUrl'],
|
|
message: `Achado de ${webSpeechPlatformLabel(finding.platform)} exige mediaUrl (arquivo direto).`,
|
|
})
|
|
}
|
|
if (webSpeechAtFromPublishedAt(finding.publishedAt) === null) {
|
|
context.addIssue({
|
|
code: 'custom',
|
|
path: ['publishedAt'],
|
|
message: 'publishedAt deve ser YYYY-MM-DD ou data/hora ISO.',
|
|
})
|
|
}
|
|
})
|
|
|
|
export type WebSpeechFinding = z.infer<typeof webSpeechFindingSchema>
|
|
|
|
export type WebSpeechFindingParseResult =
|
|
| { ok: true; finding: WebSpeechFinding }
|
|
| { ok: false; error: string }
|
|
|
|
export const parseWebSpeechFinding = (value: unknown): WebSpeechFindingParseResult => {
|
|
const parsed = webSpeechFindingSchema.safeParse(value)
|
|
if (parsed.success) return { ok: true, finding: parsed.data }
|
|
const issue = parsed.error.issues[0]
|
|
const path = issue?.path.join('.') ?? ''
|
|
return {
|
|
ok: false,
|
|
error: path ? `${path}: ${issue?.message}` : (issue?.message ?? 'achado inválido'),
|
|
}
|
|
}
|
|
|
|
type WebSpeechBatch = {
|
|
generatedAt: string | null
|
|
findings: unknown[]
|
|
}
|
|
|
|
export type WebSpeechBatchParseResult =
|
|
| { ok: true; batch: WebSpeechBatch }
|
|
| { ok: false; error: string }
|
|
|
|
/**
|
|
* Envelope only: the findings stay raw so one invalid entry becomes a failure
|
|
* of that entry in the report instead of killing the whole batch.
|
|
*/
|
|
export const parseWebSpeechBatch = (value: unknown): WebSpeechBatchParseResult => {
|
|
const envelope = z
|
|
.object({
|
|
generatedAt: z.string().trim().min(1).optional(),
|
|
findings: z.array(z.unknown()),
|
|
})
|
|
.safeParse(value)
|
|
if (!envelope.success) {
|
|
return { ok: false, error: 'lote inválido: esperado { generatedAt?, findings: [...] }.' }
|
|
}
|
|
return {
|
|
ok: true,
|
|
batch: {
|
|
generatedAt: envelope.data.generatedAt ?? null,
|
|
findings: envelope.data.findings,
|
|
},
|
|
}
|
|
}
|