/** * Seeds the `post` / `tag` / `media` collections from a LIVE fetch of * jorgesolla.com.br, classifying each article by category + tags. * * Source of truth is the live site (fetched when the script runs), NOT the * reference `noticias-jorgesolla.md`. Primary source is the public WordPress * REST API (`/wp-json/wp/v2/posts`), with an HTML category-archive crawl as a * fallback if the API is blocked/unavailable. * * Safety model (mirrors scripts/guard-dev-db.mjs and scripts/db-pull.mjs): * - This script WRITES (creates posts/tags/media). It refuses to run against * a non-local DATABASE_URL unless ALLOW_REMOTE_DB=true is set explicitly. * - Idempotent: re-running looks up posts/tags/media by slug/filename and * skips (or reuses) what already exists, so it never duplicates. * * HTML -> Lexical: the WordPress `content.rendered` HTML is converted to * Payload Lexical richText with `convertHTMLToLexical` from * `@payloadcms/richtext-lexical`, using the project's default editor config * (via `editorConfigFactory.default`) and `jsdom` as the DOM implementation. * The frontend renders `body` with `convertLexicalToHTML`, so this round-trips. * * Usage: * pnpm db:seed:posts * (against a remote DB, on purpose:) ALLOW_REMOTE_DB=true pnpm db:seed:posts */ import { convertHTMLToLexical, editorConfigFactory } from '@payloadcms/richtext-lexical' import { JSDOM } from 'jsdom' import { getPayload } from 'payload' import { dieWithLabel, loadCliEnv } from './lib/cli.mjs' import { assertLocalDatabase } from './assert-local-database.mjs' // Mirror Next.js precedence (.env.local wins over .env) without clobbering a // DATABASE_URL already set in the real environment. loadCliEnv() const config = (await import('../src/payload.config.ts')).default const die = dieWithLabel('seed:posts') // --------------------------------------------------------------------------- // Taxonomy (Section 7): the categories + control tag, and the per-article map. // Keys are the WordPress slugs of the live articles. Every post gets exactly // one required `category`; `tags` is optional (currently only `eleitoral`). // The `eleitoral` tag marks pre-candidacy / campaign / PGP / party-event // content so Francisco can hide it all with one toggle in the electoral period. // --------------------------------------------------------------------------- const TAXONOMY_TAGS = [ { slug: 'saude', name: 'Saúde' }, { slug: 'cultura', name: 'Cultura' }, { slug: 'desenvolvimento', name: 'Desenvolvimento' }, { slug: 'politica', name: 'Política' }, { slug: 'educacao', name: 'Educação' }, { slug: 'seguranca', name: 'Segurança' }, { slug: 'direitos-trabalhistas', name: 'Direitos trabalhistas' }, { slug: 'eleitoral', name: 'Eleitoral' }, ] const DEFAULT_CATEGORY = 'politica' /** WP slug -> { category, tags }. See report/plan Section 7. */ const CLASSIFICATION = { // Health 'lula-jeronimo-e-solla-inauguram-hospital-do-litoral-norte-da-bahia': { category: 'saude' }, 'solla-reforca-luta-por-cumprimento-do-piso-da-enfermagem-em-audiencia-na-camara': { category: 'saude', }, 'emenda-de-solla-garante-maquinas-de-costura-a-associacao-de-fibromialgia': { category: 'saude' }, 'solla-comemora-25-anos-da-politica-nacional-de-sangue-neste-16-de-abril': { category: 'saude' }, 'fora-trump-em-defesa-da-saude-e-da-paz': { category: 'saude' }, 'portaria-autoriza-retorno-de-mais-de-300-servidores-a-funasa': { category: 'saude' }, 'missao-internacional-com-solla-lula-e-jeronimo-fortalece-producao-de-medicamentos-estrategicos-e-projeta-a-bahia-como-polo-de-biofarmacos': { category: 'saude' }, 'solla-desembarca-na-india-em-missao-oficial-com-lula-pela-asia': { category: 'saude' }, // Culture 'solla-exalta-primeiro-ano-de-salvador-como-capital-do-brasil-no-2-de-julho': { category: 'cultura', }, 'audiencia-publica-debate-plano-nacional-de-cultura': { category: 'cultura' }, // Development 'solla-e-jeronimo-inauguram-mercado-publico-de-ubata': { category: 'desenvolvimento' }, 'governo-da-bahia-anuncia-pacote-de-quase-r-2-bi': { category: 'desenvolvimento' }, 'com-solla-comissao-acompanha-retomada-da-producao-no-enseada-em-maragogipe': { category: 'desenvolvimento', }, 'recomprar-a-refinaria-de-mataripe-e-defender-a-bahia-e-a-soberania-do-brasil': { category: 'desenvolvimento', }, 'solla-protocola-pec-para-garantir-controle-estatal-sobre-petroleo-e-derivados-no-brasil': { category: 'desenvolvimento', }, 'solla-leva-comissao-da-camara-para-nova-visita-tecnica-ao-estaleiro-enseada': { category: 'desenvolvimento', }, 'entrevista-solla-fala-sobre-a-retomada-dos-investimentos-publicos-no-pais': { category: 'desenvolvimento', }, 'obra-de-ampliacao-do-sistema-de-abastecimento-do-rio-paraguacu-itirucu-jaguaquara': { category: 'desenvolvimento', }, 'governo-lula-corrige-erro-de-antecessor-diz-solla-sobre-suspensao-da-importacao-de-cacau': { category: 'desenvolvimento', }, // Politics (with electoral control where applicable) 'programa-de-governo-participativo-reune-multidao-em-periperi': { category: 'politica', tags: ['eleitoral'], }, 'clube-2004-celebra-a-democracia-com-o-lancamento-da-pre-candidatura-de-jorge-solla-a-deputado-federal': { category: 'politica', tags: ['eleitoral'] }, 'articulada-por-solla-filiacao-de-zilar-portela-ao-pt-marca-lancamento-de-pre-candidatura-a-deputada-estadual': { category: 'politica', tags: ['eleitoral'] }, 'articulacao-de-solla-fortalece-pt-com-filiacao-da-lideranca-sindical-zilar-portela': { category: 'politica', tags: ['eleitoral'], }, 'federacao-brasil-esperanca-exige-defesa-do-nosso-projeto-politico': { category: 'politica', tags: ['eleitoral'], }, 'com-presenca-de-solla-e-aliados-aniversario-do-pt-de-jacobina-reune-liderancas-e-reafirma-compromisso-o-desenvolvimento-da-bahia': { category: 'politica', tags: ['eleitoral'] }, 'pt-de-jacobina-celebra-aniversario-com-presenca-de-jorge-solla-e-do-secretario-do-governo-lula-julio-pinheiro': { category: 'politica', tags: ['eleitoral'] }, 'solla-e-indicado-em-estudo-da-arko-advice-como-um-politico-da-elite-parlamentar': { category: 'politica', }, 'solla-rebate-ataques-de-ze-coca-e-destaca-investimentos-do-mandato-em-itirucu-e-na-regiao-de-jequie': { category: 'politica' }, // Education 'solla-conduz-cffc-em-visita-tecnica-a-primeira-faculdade-de-medicina-do-brasil': { category: 'educacao', }, 'ao-lado-de-camilo-santana-jorge-solla-acompanha-avancos-da-educacao-federal-em-barreiras-e-pocoes': { category: 'educacao' }, 'solla-articula-no-mec-avancos-para-expansao-da-educacao-superior-e-da-educacao-profissional-na-bahia': { category: 'educacao' }, 'solla-comemora-anuncio-de-construcao-de-novo-campus-federal-em-jequie': { category: 'educacao' }, 'apresentacao-do-plano-de-restauracao-arquitetonica-da-famed-ufba': { category: 'educacao' }, 'solla-participa-das-celebracoes-pelos-218-anos-da-faculdade-de-medicina-da-ufba': { category: 'educacao', }, 'solla-articula-agenda-de-dirigentes-da-ufba-no-ministerio-da-cultura-para-viabilizar-reforma-de-alas-da-faculdade-de-medicina': { category: 'educacao' }, // Security 'camara-aprova-pec-da-seguranca-publica-e-solla-destaca-avanco-da-proposta-do-governo-lula': { category: 'seguranca', }, // Labor rights 'fim-da-escala-6x1-tambem-e-questao-de-saude': { category: 'direitos-trabalhistas' }, 'jorge-solla-apresenta-pec-que-eleva-pensao-por-morte': { category: 'direitos-trabalhistas' }, 'da-asia-solla-manifesta-apoio-a-servidores-demitidos-por-bruno-reis': { category: 'direitos-trabalhistas', }, } // --------------------------------------------------------------------------- // Live fetch — WordPress REST API (primary) // --------------------------------------------------------------------------- const BASE_URL = 'https://jorgesolla.com.br' const USER_AGENT = 'teqo-content-migration-bot/1.0 (+https://jorgesolla.com.br)' const stripHtml = (html) => new JSDOM(`
${html || ''}`).window.document.body.textContent .replace(/\s+/g, ' ') .trim() /** Normalized article shape used by both the REST and HTML strategies. */ /** @typedef {{ slug: string, title: string, date: string|null, html: string, coverUrl: string|null, coverAlt: string|null }} Article */ async function fetchViaRestApi() { /** @type {Article[]} */ const articles = [] let page = 1 let totalPages = 1 do { const url = `${BASE_URL}/wp-json/wp/v2/posts?per_page=100&page=${page}&_embed` const res = await fetch(url, { headers: { 'User-Agent': USER_AGENT } }) if (!res.ok) throw new Error(`REST API returned ${res.status} for page ${page}`) if (page === 1) { totalPages = Number(res.headers.get('x-wp-totalpages') || '1') || 1 } const data = await res.json() if (!Array.isArray(data) || data.length === 0) break for (const item of data) { const embeddedMedia = item?._embedded?.['wp:featuredmedia']?.[0] articles.push({ slug: item.slug, title: stripHtml(item?.title?.rendered) || item.slug, date: item.date || null, html: item?.content?.rendered || '', coverUrl: embeddedMedia?.source_url || null, coverAlt: embeddedMedia?.alt_text || null, }) } page += 1 } while (page <= totalPages) return articles } // --------------------------------------------------------------------------- // Live fetch — HTML category-archive crawl (fallback) // Mirrors conteudo-atual-scraping/scrape_jorgesolla.py. // --------------------------------------------------------------------------- const FALLBACK_CATEGORIES = ['noticias', 'uncategorized'] const PERMALINK_RE = new RegExp( `^${BASE_URL.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/\\d{4}/\\d{2}/\\d{2}/[^/]+/?$`, ) async function fetchViaHtmlArchive() { /** @type {Article[]} */ const articles = [] const seen = new Set() for (const category of FALLBACK_CATEGORIES) { let pageNum = 1 while (pageNum <= 30) { const url = pageNum === 1 ? `${BASE_URL}/category/${category}/` : `${BASE_URL}/category/${category}/page/${pageNum}/` const res = await fetch(url, { headers: { 'User-Agent': USER_AGENT } }) if (res.status === 404) break if (!res.ok) throw new Error(`archive ${url} returned ${res.status}`) const doc = new JSDOM(await res.text()).window.document const links = new Set() for (const heading of doc.querySelectorAll('h1, h2, h3, h4, h5, h6')) { const a = heading.querySelector('a[href]') if (!a) continue const href = a.href.replace(/\/?$/, '/') if (PERMALINK_RE.test(href)) links.add(href) } const fresh = [...links].filter((href) => !seen.has(href)) if (fresh.length === 0) break for (const href of fresh) { seen.add(href) const article = await fetchSinglePage(href) if (article) articles.push(article) } pageNum += 1 } } return articles } async function fetchSinglePage(url) { const res = await fetch(url, { headers: { 'User-Agent': USER_AGENT } }) if (!res.ok) return null const doc = new JSDOM(await res.text()).window.document const slug = url.replace(/\/$/, '').split('/').pop() const contentEl = doc.querySelector('.entry-content') || doc.querySelector('article') || doc.body const time = doc.querySelector('time[datetime]') const firstImg = contentEl.querySelector('img') return { slug, title: stripHtml(doc.querySelector('h1')?.innerHTML || slug), date: time?.getAttribute('datetime') || null, html: contentEl.innerHTML || '', coverUrl: firstImg ? new URL(firstImg.getAttribute('src'), url).href : null, coverAlt: firstImg?.getAttribute('alt') || null, } } // --------------------------------------------------------------------------- // Content processing: split WP content into { subtitle, coverUrl, bodyHtml } // --------------------------------------------------------------------------- /** * WordPress articles on this site follow a consistent shape: *dek/subtitle (optional)
* immediately followed
// by the cover-image figure. Only treat the first paragraph as the subtitle
// when that exact shape holds — otherwise the first paragraph is real body.
let subtitle = null
const firstBlock = root.firstElementChild
if (firstBlock && firstBlock.tagName === 'P') {
const next = firstBlock.nextElementSibling
const nextIsImage = !!next && (next.tagName === 'IMG' || !!next.querySelector?.('img'))
const text = firstBlock.textContent.replace(/\s+/g, ' ').trim()
if (nextIsImage && text && !/^por\s+jorge\s+solla/i.test(text)) {
subtitle = text
firstBlock.remove()
}
}
// Strip ALL images (and their figure wrappers): the cover is stored separately
// on the post, and inline external images can't be resolved to `media` docs, so
// `convertHTMLToLexical` would emit invalid "pending" upload nodes. Dropping
// them keeps the body as valid text/heading/list/link/quote content.
for (const img of root.querySelectorAll('img')) {
;(img.closest('figure') || img).remove()
}
// Remove any figures/paragraphs left empty by the image removal.
for (const el of root.querySelectorAll('figure, p')) {
if (el.querySelector('img')) continue
if (el.textContent.replace(/\u00a0/g, ' ').trim() === '') el.remove()
}
return { subtitle, coverUrl, bodyHtml: root.innerHTML.trim() }
}
// ---------------------------------------------------------------------------
// Media upload (idempotent by filename)
// ---------------------------------------------------------------------------
const EXT_BY_MIME = {
'image/jpeg': '.jpg',
'image/png': '.png',
'image/webp': '.webp',
'image/gif': '.gif',
'image/avif': '.avif',
}
/** @returns {Promise<{ id: string|number, created: boolean } | null>} */
async function ensureCoverMedia(payload, { slug, coverUrl, alt }) {
if (!coverUrl) return null
const res = await fetch(coverUrl, { headers: { 'User-Agent': USER_AGENT } })
if (!res.ok) {
console.warn(`[seed:posts] cover download failed (${res.status}) for ${slug}: ${coverUrl}`)
return null
}
const mimetype = (res.headers.get('content-type') || 'image/jpeg').split(';')[0].trim()
const urlExt = new URL(coverUrl).pathname.match(/\.[a-z0-9]+$/i)?.[0]
const ext = EXT_BY_MIME[mimetype] || urlExt || '.jpg'
const filename = `${slug}${ext}`
// Idempotency: reuse an existing media doc with the same (stable) filename
// if it's already there (e.g. from a previous partial run).
const existing = await payload.find({
collection: 'media',
where: { filename: { equals: filename } },
limit: 1,
depth: 0,
})
if (existing.docs.length > 0) return { id: existing.docs[0].id, created: false }
const buffer = Buffer.from(await res.arrayBuffer())
// The upload goes through the configured media storage plugin: Garage S3
// when the S3_* envs are set (OPS52), local disk otherwise. The S3 adapter
// overwrites the deterministic key (`