mirror of
https://github.com/fsolla/teqo.git
synced 2026-11-12 22:13:05 +00:00
feat(C199-large): gravações do acervo sem o teto de 4 GB
O diálogo, a rota e o stream deixam de recusar arquivos acima de 4 GiB; o nome do arquivo e o `formatRecordingFileSize` continuam iguais. Acima de 2 GiB a mídia não passa mais pelo `fs.readFile` do Payload local API: a row nasce de um placeholder de 0 byte e a mesma key é sobrescrita por upload multipart direto no S3 (padrão C225-large, extraído para `utilities/privateMedia/privateMediaUpload.ts` e reusado pelo espelho de falas web, que mantém o teto de 5 GiB via `maxBytes` do chamador). O objeto é removido se a transação falhar, sem row fantasma, e o upload multipart de gravações não tem teto (`Infinity`).
This commit is contained in:
parent
671af1e21f
commit
4c632d43bf
12 changed files with 442 additions and 159 deletions
1
docs/changelog/2026-09-26-gravacoes-sem-teto-4gb.md
Normal file
1
docs/changelog/2026-09-26-gravacoes-sem-teto-4gb.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
**Gravações enviadas ao acervo sem o teto de 4 GB (2026-09-26):** o diálogo "Enviar gravação" recusava qualquer arquivo acima de 4 GiB (`RECORDING_MAX_BYTES`) — limite que plenárias longas encostam. O teto saiu do cliente, da rota e do stream: o corpo continua indo para o temp com backpressure, agora sem teto de produto (disco e multipart do S3 são o limite). Acima de 2 GiB a mídia deixa de passar pelo `fs.readFile` do Payload: o `recordingMedia` nasce de um placeholder de 0 byte e a mesma key é sobrescrita por upload multipart direto no S3 — padrão do C225-large, extraído para o dono `utilities/privateMedia/privateMediaUpload.ts` e agora compartilhado com o espelho de falas web (que mantém o próprio teto de 5 GiB via `maxBytes` do chamador; gravações passam `Infinity`). O nome original do arquivo sobrevive ao placeholder e o objeto é removido se a transação falhar, sem row fantasma. Sem migration e sem schema; unit (novo helper + limites do uploader) e int de gravações/falas web verdes. Operação: o `/tmp` do container precisa de headroom para o tamanho da gravação (upload + download do job de transcrição).
|
||||
|
|
@ -7,7 +7,6 @@ import {
|
|||
RECORDING_FILE_TYPE_MESSAGE,
|
||||
RECORDING_TITLE_REQUIRED_MESSAGE,
|
||||
recordingFileTypeAllowed,
|
||||
recordingUploadTooLargeMessage,
|
||||
toRecordingViewModel,
|
||||
} from '@/lib/recording'
|
||||
import {
|
||||
|
|
@ -49,7 +48,6 @@ const SAFE_MESSAGES = [
|
|||
RECORDING_TITLE_REQUIRED_MESSAGE,
|
||||
RECORDING_FILE_NAME_MESSAGE,
|
||||
RECORDING_FILE_TYPE_MESSAGE,
|
||||
recordingUploadTooLargeMessage,
|
||||
RECORDING_BODY_MISSING_MESSAGE,
|
||||
]
|
||||
|
||||
|
|
@ -78,13 +76,6 @@ export const POST = async (request: Request): Promise<NextResponse<RecordingUplo
|
|||
return errorResponse(RECORDING_FILE_TYPE_MESSAGE, 400)
|
||||
}
|
||||
|
||||
const contentLengthHeader = request.headers.get('content-length')
|
||||
const parsedLength = contentLengthHeader === null ? null : Number(contentLengthHeader)
|
||||
const contentLength =
|
||||
parsedLength !== null && Number.isFinite(parsedLength) && parsedLength >= 0
|
||||
? parsedLength
|
||||
: null
|
||||
|
||||
try {
|
||||
const payload = await getPayload({ config })
|
||||
const { id } = await receiveRecordingUpload({
|
||||
|
|
@ -92,7 +83,6 @@ export const POST = async (request: Request): Promise<NextResponse<RecordingUplo
|
|||
actor: user,
|
||||
metadata: parsed.data,
|
||||
body: request.body,
|
||||
contentLength,
|
||||
})
|
||||
|
||||
const recording = await payload.findByID({
|
||||
|
|
|
|||
|
|
@ -24,12 +24,9 @@ import {
|
|||
formatRecordingFileSize,
|
||||
RECORDING_FILE_REQUIRED_MESSAGE,
|
||||
RECORDING_FILE_TYPE_MESSAGE,
|
||||
RECORDING_MAX_BYTES,
|
||||
RECORDING_MAX_SIZE_LABEL,
|
||||
RECORDING_TITLE_MAX_LENGTH,
|
||||
RECORDING_TITLE_REQUIRED_MESSAGE,
|
||||
recordingFileTypeAllowed,
|
||||
recordingTooLargeMessage,
|
||||
} from '@/lib/recording'
|
||||
import { cn } from '@/lib/utils'
|
||||
|
||||
|
|
@ -109,10 +106,6 @@ export const RecordingUploadDialog = ({
|
|||
setFile(next)
|
||||
if (!recordingFileTypeAllowed(next.type)) {
|
||||
setFileError(RECORDING_FILE_TYPE_MESSAGE)
|
||||
return
|
||||
}
|
||||
if (next.size > RECORDING_MAX_BYTES) {
|
||||
setFileError(recordingTooLargeMessage(next.size))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -285,7 +278,7 @@ export const RecordingUploadDialog = ({
|
|||
<UploadIcon className="size-6 text-muted-foreground" aria-hidden="true" />
|
||||
<span className="mt-2 text-sm font-medium">Selecionar arquivo local</span>
|
||||
<span className="mt-1 text-xs text-muted-foreground">
|
||||
Vídeos até {RECORDING_MAX_SIZE_LABEL} · MP4, MOV, MKV e formatos compatíveis
|
||||
MP4, MOV, MKV e formatos compatíveis
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
|
|
@ -297,7 +290,7 @@ export const RecordingUploadDialog = ({
|
|||
<AlertDescription className="text-xs">{fileError}</AlertDescription>
|
||||
</Alert>
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
Vídeos até {RECORDING_MAX_SIZE_LABEL} · MP4, MOV, MKV e formatos compatíveis
|
||||
MP4, MOV, MKV e formatos compatíveis
|
||||
</p>
|
||||
</>
|
||||
) : null}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* C199 — pure rules of an uploaded recording: the status/step vocabulary, the
|
||||
* upload limits and the honest failure copy. No I/O and no `server-only`: the
|
||||
* client dialog, the JSON routes, the job and the unit tests share this module.
|
||||
* C199 — pure rules of an uploaded recording: the status/step vocabulary and
|
||||
* the honest failure copy. No I/O and no `server-only`: the client dialog, the
|
||||
* JSON routes, the job and the unit tests share this module.
|
||||
*/
|
||||
import {
|
||||
CAMPAIGN_COMMUNICATION_ACERVO_GRAVACOES,
|
||||
|
|
@ -35,9 +35,7 @@ export const RECORDING_SPEAKER_LABEL_MAX_LENGTH = 120
|
|||
export const RECORDING_SPEAKER_LABEL_REQUIRED_MESSAGE = 'Informe o nome do falante.'
|
||||
export const RECORDING_SPEAKER_LABEL_LONG_MESSAGE = `O nome do falante deve ter até ${RECORDING_SPEAKER_LABEL_MAX_LENGTH} caracteres.`
|
||||
|
||||
/** Upload ceiling: hours of plenary are the real case, 4 GiB is the guard. */
|
||||
export const RECORDING_MAX_BYTES = 4 * 1024 * 1024 * 1024
|
||||
export const RECORDING_MAX_SIZE_LABEL = '4 GB'
|
||||
/** Accepted upload type: any video container (`.mkv` often arrives with no type). */
|
||||
const RECORDING_ACCEPTED_MIME_PREFIX = 'video/'
|
||||
|
||||
export const RECORDING_STEPS = ['extracting', 'transcribing', 'saving'] as const
|
||||
|
|
@ -147,11 +145,6 @@ export const formatRecordingFileSize = (bytes: number): string => {
|
|||
return `${Math.round(bytes)} B`
|
||||
}
|
||||
|
||||
export const recordingTooLargeMessage = (bytes: number): string =>
|
||||
`Este arquivo tem ${formatRecordingFileSize(bytes)}. Escolha um vídeo de até ${RECORDING_MAX_SIZE_LABEL}.`
|
||||
|
||||
export const recordingUploadTooLargeMessage = `O arquivo excede o limite de ${RECORDING_MAX_SIZE_LABEL}.`
|
||||
|
||||
export type RecordingViewModel = {
|
||||
id: number
|
||||
title: string
|
||||
|
|
|
|||
|
|
@ -17,7 +17,10 @@ const MIME_BY_EXTENSION: Record<string, string> = {
|
|||
'.aac': 'audio/aac',
|
||||
'.flac': 'audio/flac',
|
||||
'.m4a': 'audio/mp4',
|
||||
'.m4v': 'video/x-m4v',
|
||||
'.mka': 'audio/x-matroska',
|
||||
'.mkv': 'video/x-matroska',
|
||||
'.mov': 'video/quicktime',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.mp4': 'video/mp4',
|
||||
'.ogg': 'audio/ogg',
|
||||
|
|
@ -116,18 +119,21 @@ export const uploadLargeMedia = async ({
|
|||
inputPath,
|
||||
filename: providedFilename,
|
||||
env = process.env,
|
||||
maxBytes = MAX_LARGE_MEDIA_BYTES,
|
||||
upload = defaultUpload,
|
||||
}: {
|
||||
inputPath: string
|
||||
filename?: string
|
||||
env?: Record<string, string | undefined>
|
||||
/** Caller ceiling; `Infinity` drops it (the S3 multipart limit is the bound). */
|
||||
maxBytes?: number
|
||||
upload?: (args: S3UploadArgs) => Promise<void>
|
||||
}): Promise<LargeMediaUpload> => {
|
||||
const storage = requireStorage(env)
|
||||
const { size } = await stat(inputPath)
|
||||
if (size > MAX_LARGE_MEDIA_BYTES) {
|
||||
if (size > maxBytes) {
|
||||
throw new Error(
|
||||
`Mídia de ${(size / 1024 / 1024 / 1024).toFixed(2)} GiB excede o limite de 5 GiB.`,
|
||||
`Mídia de ${(size / 1024 / 1024 / 1024).toFixed(2)} GiB excede o limite de ${(maxBytes / 1024 / 1024 / 1024).toFixed(0)} GiB.`,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
95
src/utilities/privateMedia/privateMediaUpload.ts
Normal file
95
src/utilities/privateMedia/privateMediaUpload.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import 'server-only'
|
||||
|
||||
import { mkdir, stat, writeFile } from 'node:fs/promises'
|
||||
import { dirname, extname, join } from 'node:path'
|
||||
|
||||
import {
|
||||
deleteLargeMedia,
|
||||
isLargeMedia,
|
||||
LARGE_MEDIA_THRESHOLD_BYTES,
|
||||
uploadLargeMedia,
|
||||
} from '@/utilities/media/largeS3Upload'
|
||||
|
||||
/**
|
||||
* C225-large/C199 — creates one private upload row from a local file without
|
||||
* ever letting Payload's local API `fs.readFile` an arbitrarily large artifact:
|
||||
* above the buffer boundary the row is created through an empty placeholder and
|
||||
* the same object key is overwritten by a direct multipart stream, then the
|
||||
* stored `filesize`/MIME metadata is updated. Below the boundary the Payload
|
||||
* path is used as-is. The caller's `create`/`update` closures own collection,
|
||||
* access and transaction details.
|
||||
*
|
||||
* `cleanup` removes the multipart object; call it when the surrounding
|
||||
* transaction fails, since the row is rolled back with it.
|
||||
*/
|
||||
|
||||
export type PrivateMediaFromFile = {
|
||||
id: number
|
||||
cleanup: () => Promise<void>
|
||||
}
|
||||
|
||||
type PrivateMediaRow = { id: number; filename?: string | null }
|
||||
|
||||
export type CreatePrivateMediaFromFileArgs = {
|
||||
inputPath: string
|
||||
/**
|
||||
* Placeholder file the large path creates the row from; its basename is the
|
||||
* stored name Payload derives. Override it to keep the original file name.
|
||||
*/
|
||||
placeholderPath?: string
|
||||
create: (filePath: string) => Promise<PrivateMediaRow>
|
||||
update: (id: number, data: { filesize: number; mimeType: string }) => Promise<unknown>
|
||||
thresholdBytes?: number
|
||||
/** Multipart ceiling; omit to keep the uploader default. */
|
||||
maxBytes?: number
|
||||
uploadLarge?: typeof uploadLargeMedia
|
||||
removeLarge?: typeof deleteLargeMedia
|
||||
}
|
||||
|
||||
const noopCleanup = async (): Promise<void> => {}
|
||||
|
||||
export const createPrivateMediaFromFile = async ({
|
||||
inputPath,
|
||||
placeholderPath,
|
||||
create,
|
||||
update,
|
||||
thresholdBytes = LARGE_MEDIA_THRESHOLD_BYTES,
|
||||
maxBytes,
|
||||
uploadLarge = uploadLargeMedia,
|
||||
removeLarge = deleteLargeMedia,
|
||||
}: CreatePrivateMediaFromFileArgs): Promise<PrivateMediaFromFile> => {
|
||||
const { size } = await stat(inputPath)
|
||||
if (!isLargeMedia(size, thresholdBytes)) {
|
||||
const media = await create(inputPath)
|
||||
return { id: media.id, cleanup: noopCleanup }
|
||||
}
|
||||
|
||||
const placeholder =
|
||||
placeholderPath ?? join(dirname(inputPath), `payload-placeholder${extname(inputPath)}`)
|
||||
await mkdir(dirname(placeholder), { recursive: true })
|
||||
await writeFile(placeholder, Buffer.alloc(0))
|
||||
|
||||
let uploadedFilename: string | null = null
|
||||
try {
|
||||
const media = await create(placeholder)
|
||||
uploadedFilename = media.filename ?? null
|
||||
if (!uploadedFilename) throw new Error('A mídia grande não recebeu um filename no S3.')
|
||||
const uploaded = await uploadLarge({
|
||||
inputPath,
|
||||
filename: uploadedFilename,
|
||||
...(maxBytes === undefined ? {} : { maxBytes }),
|
||||
})
|
||||
await update(media.id, { filesize: uploaded.filesize, mimeType: uploaded.mimeType })
|
||||
|
||||
const filename = uploadedFilename
|
||||
return {
|
||||
id: media.id,
|
||||
cleanup: async () => {
|
||||
await removeLarge({ filename }).catch(() => undefined)
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
if (uploadedFilename) await removeLarge({ filename: uploadedFilename }).catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
|
@ -4,65 +4,45 @@ import { createWriteStream } from 'node:fs'
|
|||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Readable, Transform } from 'node:stream'
|
||||
import { Readable } from 'node:stream'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import type { Payload } from 'payload'
|
||||
|
||||
import {
|
||||
RECORDING_MAX_BYTES,
|
||||
RECORDING_MEDIA_SLUG,
|
||||
recordingUploadTooLargeMessage,
|
||||
sanitizeRecordingFilename,
|
||||
} from '@/lib/recording'
|
||||
import { RECORDING_MEDIA_SLUG, sanitizeRecordingFilename } from '@/lib/recording'
|
||||
import type { RecordingUploadMetadata } from '@/lib/schemas/recording'
|
||||
import type { CampaignUser } from '@/payload-types'
|
||||
import { deleteLargeMedia, uploadLargeMedia } from '@/utilities/media/largeS3Upload'
|
||||
import { withPayloadTransaction } from '@/utilities/payloadTransaction'
|
||||
import { createPrivateMediaFromFile } from '@/utilities/privateMedia/privateMediaUpload'
|
||||
import { startRecordingJobInBackground } from '@/utilities/recordings/recordingScheduler'
|
||||
|
||||
/**
|
||||
* C199 — the upload half of the recordings flow: create the visible row, stream
|
||||
* the raw request body to a temp file with the size ceiling enforced, then store
|
||||
* the private media and hand the row to the transcription job. The route owns
|
||||
* the origin/session/role gate; this module owns the write path.
|
||||
* the raw request body to a temp file, then store the private media (multipart
|
||||
* above the Payload buffer boundary, see `privateMediaUpload`) and hand the row
|
||||
* to the transcription job. The route owns the origin/session/role gate; this
|
||||
* module owns the write path.
|
||||
*/
|
||||
|
||||
/** Shared literal: the route allowlists it and the stream guard throws it. */
|
||||
/** Shared literal: the route allowlists it and the missing body throws it. */
|
||||
export const RECORDING_BODY_MISSING_MESSAGE = 'A requisição não trouxe o arquivo da gravação.'
|
||||
|
||||
const toUploadTooLarge = (): Error => new Error(recordingUploadTooLargeMessage)
|
||||
|
||||
/**
|
||||
* Streams the request body to `destination`, enforcing the ceiling even when
|
||||
* the request arrives chunked (a missing/lying `Content-Length` cannot fill the
|
||||
* disk). Returns the received byte count.
|
||||
* Streams the request body to `destination` with backpressure. There is no
|
||||
* product ceiling (the upload accepts hours of plenary): the container disk and
|
||||
* the S3 multipart limit are the bounds.
|
||||
*/
|
||||
const streamBodyToFile = async ({
|
||||
body,
|
||||
destination,
|
||||
maxBytes,
|
||||
}: {
|
||||
body: ReadableStream<Uint8Array>
|
||||
destination: string
|
||||
maxBytes: number
|
||||
}): Promise<number> => {
|
||||
let received = 0
|
||||
const guard = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
received += chunk.length
|
||||
if (received > maxBytes) {
|
||||
callback(toUploadTooLarge())
|
||||
return
|
||||
}
|
||||
callback(null, chunk)
|
||||
},
|
||||
})
|
||||
|
||||
}): Promise<void> => {
|
||||
await pipeline(
|
||||
Readable.fromWeb(body as unknown as import('node:stream/web').ReadableStream),
|
||||
guard,
|
||||
createWriteStream(destination),
|
||||
)
|
||||
return received
|
||||
}
|
||||
|
||||
export const receiveRecordingUpload = async ({
|
||||
|
|
@ -70,18 +50,22 @@ export const receiveRecordingUpload = async ({
|
|||
actor,
|
||||
metadata,
|
||||
body,
|
||||
contentLength,
|
||||
startJob = startRecordingJobInBackground,
|
||||
largeThresholdBytes,
|
||||
uploadLarge,
|
||||
removeLarge,
|
||||
}: {
|
||||
payload: Payload
|
||||
actor: CampaignUser
|
||||
metadata: RecordingUploadMetadata
|
||||
body: ReadableStream<Uint8Array> | null
|
||||
contentLength: number | null
|
||||
/** Injectable for tests; the route uses the `after()`-based default. */
|
||||
startJob?: (recordingId: number) => void
|
||||
/** Test seams; the production defaults live in the private-media owner. */
|
||||
largeThresholdBytes?: number
|
||||
uploadLarge?: typeof uploadLargeMedia
|
||||
removeLarge?: typeof deleteLargeMedia
|
||||
}): Promise<{ id: number }> => {
|
||||
if (contentLength !== null && contentLength > RECORDING_MAX_BYTES) throw toUploadTooLarge()
|
||||
if (!body) throw new Error(RECORDING_BODY_MISSING_MESSAGE)
|
||||
|
||||
const recording = await payload.create({
|
||||
|
|
@ -97,22 +81,48 @@ export const receiveRecordingUpload = async ({
|
|||
})
|
||||
|
||||
let tempDir: string | null = null
|
||||
const mediaCleanup: { current?: () => Promise<void> } = {}
|
||||
try {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'recording-upload-'))
|
||||
const uploadDir = await mkdtemp(join(tmpdir(), 'recording-upload-'))
|
||||
tempDir = uploadDir
|
||||
const uploadName = sanitizeRecordingFilename(metadata.filename)
|
||||
const tempPath = join(tempDir, uploadName)
|
||||
await streamBodyToFile({ body, destination: tempPath, maxBytes: RECORDING_MAX_BYTES })
|
||||
const tempPath = join(uploadDir, uploadName)
|
||||
await streamBodyToFile({ body, destination: tempPath })
|
||||
|
||||
await withPayloadTransaction(payload, async ({ req }) => {
|
||||
const media = await payload.create({
|
||||
collection: RECORDING_MEDIA_SLUG,
|
||||
data: { alt: metadata.title },
|
||||
filePath: tempPath,
|
||||
depth: 0,
|
||||
user: actor,
|
||||
overrideAccess: false,
|
||||
req,
|
||||
const media = await createPrivateMediaFromFile({
|
||||
inputPath: tempPath,
|
||||
// Same basename as the upload: the stored name (and the download) keeps
|
||||
// the original file name even on the multipart path.
|
||||
placeholderPath: join(uploadDir, 'placeholder', uploadName),
|
||||
thresholdBytes: largeThresholdBytes,
|
||||
// No product ceiling for recordings; disk is the bound.
|
||||
maxBytes: Number.POSITIVE_INFINITY,
|
||||
uploadLarge,
|
||||
removeLarge,
|
||||
create: (filePath) =>
|
||||
payload.create({
|
||||
collection: RECORDING_MEDIA_SLUG,
|
||||
data: { alt: metadata.title },
|
||||
filePath,
|
||||
depth: 0,
|
||||
user: actor,
|
||||
overrideAccess: false,
|
||||
req,
|
||||
}),
|
||||
update: (id, data) =>
|
||||
payload.update({
|
||||
collection: RECORDING_MEDIA_SLUG,
|
||||
id,
|
||||
data,
|
||||
depth: 0,
|
||||
user: actor,
|
||||
overrideAccess: false,
|
||||
req,
|
||||
}),
|
||||
})
|
||||
mediaCleanup.current = media.cleanup
|
||||
|
||||
await payload.update({
|
||||
collection: 'recording',
|
||||
id: recording.id,
|
||||
|
|
@ -123,12 +133,16 @@ export const receiveRecordingUpload = async ({
|
|||
req,
|
||||
})
|
||||
})
|
||||
// Committed: the multipart object is the recording's media now.
|
||||
mediaCleanup.current = undefined
|
||||
|
||||
startJob(recording.id)
|
||||
return { id: recording.id }
|
||||
} catch (error) {
|
||||
// The upload never completed: the row must not linger as a phantom
|
||||
// "Enviando". The media (when it exists) is unreachable without the row.
|
||||
// "Enviando". A rolled-back transaction leaves the multipart object (when
|
||||
// the large path ran) with no row owning it, so it is removed too.
|
||||
await mediaCleanup.current?.().catch(() => undefined)
|
||||
await payload
|
||||
.delete({
|
||||
collection: 'recording',
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import 'server-only'
|
||||
|
||||
import { mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, extname, join } from 'node:path'
|
||||
import { join } from 'node:path'
|
||||
import type { Payload } from 'payload'
|
||||
|
||||
import {
|
||||
|
|
@ -32,10 +32,8 @@ import { downloadUrlToFile } from '@/utilities/media/downloadToFile'
|
|||
import { ffmpegBinary, messageOf, runFfmpeg } from '@/utilities/media/ffmpeg'
|
||||
import {
|
||||
deleteLargeMedia,
|
||||
isLargeMedia,
|
||||
LARGE_MEDIA_THRESHOLD_BYTES,
|
||||
uploadLargeMedia,
|
||||
type LargeMediaUpload,
|
||||
} from '@/utilities/media/largeS3Upload'
|
||||
import {
|
||||
downloadWithYtDlp,
|
||||
|
|
@ -48,6 +46,7 @@ import {
|
|||
withPayloadTransaction,
|
||||
type PayloadTransactionRequest,
|
||||
} from '@/utilities/payloadTransaction'
|
||||
import { createPrivateMediaFromFile } from '@/utilities/privateMedia/privateMediaUpload'
|
||||
import {
|
||||
classifySpeech,
|
||||
type SpeechClassificationResult,
|
||||
|
|
@ -293,8 +292,6 @@ const extractTranscript = async ({
|
|||
return { segments, durationSeconds, asrSeconds }
|
||||
}
|
||||
|
||||
const noopCleanup = async (): Promise<void> => {}
|
||||
|
||||
type WebSpeechMediaMirrorOptions = {
|
||||
thresholdBytes?: number
|
||||
uploadLarge?: typeof uploadLargeMedia
|
||||
|
|
@ -307,60 +304,31 @@ export const createWebSpeechMediaMirror =
|
|||
uploadLarge = uploadLargeMedia,
|
||||
removeLarge = deleteLargeMedia,
|
||||
}: WebSpeechMediaMirrorOptions = {}): WebSpeechMediaMirror =>
|
||||
async ({ payload, req, alt, inputPath }) => {
|
||||
const { size } = await stat(inputPath)
|
||||
if (!isLargeMedia(size, thresholdBytes)) {
|
||||
const media = await payload.create({
|
||||
collection: INTERNET_SPEECH_MEDIA_SLUG,
|
||||
data: { alt },
|
||||
filePath: inputPath,
|
||||
req,
|
||||
// Intentional bypass: the ingestion CLI is a trusted actor with no session.
|
||||
overrideAccess: true,
|
||||
})
|
||||
return { id: media.id, cleanup: noopCleanup }
|
||||
}
|
||||
|
||||
const placeholderPath = join(dirname(inputPath), `payload-placeholder${extname(inputPath)}`)
|
||||
await writeFile(placeholderPath, Buffer.alloc(0))
|
||||
let filename: string | null = null
|
||||
|
||||
try {
|
||||
const media = await payload.create({
|
||||
collection: INTERNET_SPEECH_MEDIA_SLUG,
|
||||
data: { alt },
|
||||
filePath: placeholderPath,
|
||||
req,
|
||||
// Intentional bypass: the placeholder row is owned by the trusted ingestion CLI.
|
||||
overrideAccess: true,
|
||||
})
|
||||
filename = media.filename ?? null
|
||||
if (!filename) throw new Error('A mídia grande não recebeu um filename no S3.')
|
||||
const uploadedFilename = filename
|
||||
|
||||
const uploaded: LargeMediaUpload = await uploadLarge({
|
||||
inputPath,
|
||||
filename: uploadedFilename,
|
||||
})
|
||||
await payload.update({
|
||||
collection: INTERNET_SPEECH_MEDIA_SLUG,
|
||||
id: media.id,
|
||||
data: { filesize: uploaded.filesize, mimeType: uploaded.mimeType },
|
||||
req,
|
||||
// Intentional bypass: the trusted ingestion CLI finalizes its own media row.
|
||||
overrideAccess: true,
|
||||
})
|
||||
return {
|
||||
id: media.id,
|
||||
cleanup: async () => {
|
||||
await removeLarge({ filename: uploadedFilename }).catch(() => undefined)
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
if (filename) await removeLarge({ filename }).catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
({ payload, req, alt, inputPath }) =>
|
||||
createPrivateMediaFromFile({
|
||||
inputPath,
|
||||
thresholdBytes,
|
||||
uploadLarge,
|
||||
removeLarge,
|
||||
create: (filePath) =>
|
||||
payload.create({
|
||||
collection: INTERNET_SPEECH_MEDIA_SLUG,
|
||||
data: { alt },
|
||||
filePath,
|
||||
req,
|
||||
// Intentional bypass: the ingestion CLI is a trusted actor with no session.
|
||||
overrideAccess: true,
|
||||
}),
|
||||
update: (id, data) =>
|
||||
payload.update({
|
||||
collection: INTERNET_SPEECH_MEDIA_SLUG,
|
||||
id,
|
||||
data,
|
||||
req,
|
||||
// Intentional bypass: the trusted ingestion CLI finalizes its own media row.
|
||||
overrideAccess: true,
|
||||
}),
|
||||
})
|
||||
|
||||
const defaultMirrorMedia = createWebSpeechMediaMirror()
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import {
|
|||
retryRecordingForActor,
|
||||
} from '@/app/(campaign)/campanha/actions/recording'
|
||||
import { getMunicipalityCatalogEntry } from '@/lib/municipalityCatalog'
|
||||
import { RECORDING_MAX_BYTES } from '@/lib/recording'
|
||||
import {
|
||||
RECORDING_FORBIDDEN_MESSAGE,
|
||||
RECORDING_SPEAKER_UNKNOWN_MESSAGE,
|
||||
|
|
@ -345,7 +344,6 @@ describe('uploaded recordings (C199)', () => {
|
|||
recordedAt: undefined,
|
||||
},
|
||||
body,
|
||||
contentLength: bytes.length,
|
||||
startJob: (recordingId) => scheduled.push(recordingId),
|
||||
})
|
||||
createdRecordingIds.add(id)
|
||||
|
|
@ -375,31 +373,97 @@ describe('uploaded recordings (C199)', () => {
|
|||
expect(Buffer.from(await response.arrayBuffer())).toEqual(bytes)
|
||||
})
|
||||
|
||||
it('refuses an upload above the ceiling without leaving a phantom row', async () => {
|
||||
it('accepts a recording above the payload buffer boundary through the multipart seam', async () => {
|
||||
const bytes = Buffer.from('plenaria-de-horas')
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(Buffer.alloc(8))
|
||||
controller.enqueue(bytes)
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
const scheduled: number[] = []
|
||||
const uploaded: { filename?: string; filesize: number }[] = []
|
||||
const removed: string[] = []
|
||||
|
||||
const { id } = await receiveRecordingUpload({
|
||||
payload,
|
||||
actor: communicator,
|
||||
metadata: { title: 'Sessão longa', filename: 'sessao longa.MP4', recordedAt: undefined },
|
||||
body,
|
||||
largeThresholdBytes: 0,
|
||||
startJob: (recordingId) => scheduled.push(recordingId),
|
||||
uploadLarge: async ({ filename }) => {
|
||||
uploaded.push({ filename, filesize: bytes.length })
|
||||
return {
|
||||
filename: filename ?? 'sessao_longa.MP4',
|
||||
mimeType: 'video/mp4',
|
||||
filesize: bytes.length,
|
||||
url: 'http://127.0.0.1:3900/teqo-media/sessao_longa.MP4',
|
||||
}
|
||||
},
|
||||
removeLarge: async ({ filename }) => {
|
||||
removed.push(filename)
|
||||
},
|
||||
})
|
||||
createdRecordingIds.add(id)
|
||||
|
||||
const recording = await payload.findByID({
|
||||
collection: 'recording',
|
||||
id,
|
||||
depth: 1,
|
||||
overrideAccess: true,
|
||||
})
|
||||
expect(recording.status).toBe('processing')
|
||||
expect(recording.step).toBe('extracting')
|
||||
expect(scheduled).toEqual([id])
|
||||
|
||||
const media = recording.media
|
||||
expect(media && typeof media === 'object').toBe(true)
|
||||
if (media && typeof media === 'object') {
|
||||
createdMediaIds.add(media.id)
|
||||
// The original file name survives the placeholder (never the placeholder name).
|
||||
expect(media.filename).toContain('sessao_longa')
|
||||
expect(media.filesize).toBe(bytes.length)
|
||||
expect(media.mimeType).toBe('video/mp4')
|
||||
expect(uploaded).toEqual([{ filename: media.filename, filesize: bytes.length }])
|
||||
}
|
||||
expect(removed).toEqual([])
|
||||
})
|
||||
|
||||
it('removes the multipart object and leaves no phantom row when the large upload fails', async () => {
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(Buffer.from('falha-grande'))
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
const removed: string[] = []
|
||||
|
||||
await expect(
|
||||
receiveRecordingUpload({
|
||||
payload,
|
||||
actor: communicator,
|
||||
metadata: { title: 'Grande demais', filename: 'grande.mp4', recordedAt: undefined },
|
||||
metadata: { title: 'Grande que falha', filename: 'falha.mp4', recordedAt: undefined },
|
||||
body,
|
||||
contentLength: RECORDING_MAX_BYTES + 1,
|
||||
largeThresholdBytes: 0,
|
||||
startJob: () => undefined,
|
||||
uploadLarge: async () => {
|
||||
throw new Error('S3 indisponível.')
|
||||
},
|
||||
removeLarge: async ({ filename }) => {
|
||||
removed.push(filename)
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('O arquivo excede o limite de 4 GB.')
|
||||
).rejects.toThrow('S3 indisponível.')
|
||||
|
||||
const rows = await payload.find({
|
||||
collection: 'recording',
|
||||
where: { title: { equals: 'Grande demais' } },
|
||||
where: { title: { equals: 'Grande que falha' } },
|
||||
overrideAccess: true,
|
||||
})
|
||||
expect(rows.docs).toHaveLength(0)
|
||||
expect(removed).toHaveLength(1)
|
||||
expect(removed[0]).toContain('falha')
|
||||
})
|
||||
|
||||
it('preserves a long recording transcript and its search text', async () => {
|
||||
|
|
|
|||
|
|
@ -40,9 +40,36 @@ describe('large S3 media upload', () => {
|
|||
|
||||
it('maps the media extension to a safe content type', () => {
|
||||
expect(mimeTypeForPath('/tmp/source.mp4')).toBe('video/mp4')
|
||||
expect(mimeTypeForPath('/tmp/source.mov')).toBe('video/quicktime')
|
||||
expect(mimeTypeForPath('/tmp/source.mkv')).toBe('video/x-matroska')
|
||||
expect(mimeTypeForPath('/tmp/source.m4v')).toBe('video/x-m4v')
|
||||
expect(mimeTypeForPath('/tmp/source.unknown')).toBe('application/octet-stream')
|
||||
})
|
||||
|
||||
it('honors the default ceiling and an explicit caller override', async () => {
|
||||
await withTempFile(async (path) => {
|
||||
await expect(
|
||||
uploadLargeMedia({
|
||||
inputPath: path,
|
||||
env: s3Env,
|
||||
maxBytes: 8,
|
||||
upload: async () => undefined,
|
||||
}),
|
||||
).rejects.toThrow('excede o limite')
|
||||
|
||||
const keys: string[] = []
|
||||
await uploadLargeMedia({
|
||||
inputPath: path,
|
||||
env: s3Env,
|
||||
maxBytes: Number.POSITIVE_INFINITY,
|
||||
upload: async ({ key }) => {
|
||||
keys.push(key)
|
||||
},
|
||||
})
|
||||
expect(keys).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
it('streams a large file to S3 and returns row metadata', async () => {
|
||||
await withTempFile(async (path) => {
|
||||
const calls: { key: string; contentType: string; filesize: number }[] = []
|
||||
|
|
|
|||
140
tests/unit/privateMediaUpload.unit.spec.ts
Normal file
140
tests/unit/privateMediaUpload.unit.spec.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
// @vitest-environment node
|
||||
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { createPrivateMediaFromFile } from '@/utilities/privateMedia/privateMediaUpload'
|
||||
|
||||
const withTempFile = async (
|
||||
bytes: Buffer,
|
||||
run: (path: string, dir: string) => Promise<void>,
|
||||
): Promise<void> => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'private-media-upload-test-'))
|
||||
try {
|
||||
const path = join(dir, 'source-video.mp4')
|
||||
await writeFile(path, bytes)
|
||||
await run(path, dir)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
describe('private media from a local file', () => {
|
||||
it('uses the Payload path below the buffer boundary', async () => {
|
||||
await withTempFile(Buffer.alloc(16), async (path) => {
|
||||
const created: string[] = []
|
||||
let uploads = 0
|
||||
const result = await createPrivateMediaFromFile({
|
||||
inputPath: path,
|
||||
thresholdBytes: 1024,
|
||||
create: async (filePath) => {
|
||||
created.push(filePath)
|
||||
return { id: 7, filename: 'source-video.mp4' }
|
||||
},
|
||||
update: async () => undefined,
|
||||
uploadLarge: async () => {
|
||||
uploads += 1
|
||||
throw new Error('não deveria subir em multipart')
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.id).toBe(7)
|
||||
expect(created).toEqual([path])
|
||||
expect(uploads).toBe(0)
|
||||
await result.cleanup()
|
||||
})
|
||||
})
|
||||
|
||||
it('creates the row from a placeholder and overwrites it through the multipart seam', async () => {
|
||||
await withTempFile(Buffer.alloc(16), async (path, dir) => {
|
||||
const created: string[] = []
|
||||
const updated: { id: number; data: { filesize: number; mimeType: string } }[] = []
|
||||
const uploaded: { inputPath: string; filename?: string; maxBytes?: number }[] = []
|
||||
const removed: string[] = []
|
||||
|
||||
const result = await createPrivateMediaFromFile({
|
||||
inputPath: path,
|
||||
placeholderPath: join(dir, 'placeholder', 'meu-video.mp4'),
|
||||
thresholdBytes: 0,
|
||||
maxBytes: Number.POSITIVE_INFINITY,
|
||||
create: async (filePath) => {
|
||||
created.push(filePath)
|
||||
return { id: 42, filename: 'meu-video.mp4' }
|
||||
},
|
||||
update: async (id, data) => {
|
||||
updated.push({ id, data })
|
||||
return { id }
|
||||
},
|
||||
uploadLarge: async (args) => {
|
||||
uploaded.push(args)
|
||||
return {
|
||||
filename: args.filename ?? 'meu-video.mp4',
|
||||
mimeType: 'video/mp4',
|
||||
filesize: 16,
|
||||
url: 'http://127.0.0.1:3900/teqo-media/meu-video.mp4',
|
||||
}
|
||||
},
|
||||
removeLarge: async ({ filename }) => {
|
||||
removed.push(filename)
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.id).toBe(42)
|
||||
expect(created[0]).toContain(join('placeholder', 'meu-video.mp4'))
|
||||
expect((await stat(created[0])).size).toBe(0)
|
||||
expect(await readFile(created[0])).toEqual(Buffer.alloc(0))
|
||||
expect(uploaded).toEqual([
|
||||
{ inputPath: path, filename: 'meu-video.mp4', maxBytes: Number.POSITIVE_INFINITY },
|
||||
])
|
||||
expect(updated).toEqual([{ id: 42, data: { filesize: 16, mimeType: 'video/mp4' } }])
|
||||
expect(removed).toEqual([])
|
||||
|
||||
await result.cleanup()
|
||||
expect(removed).toEqual(['meu-video.mp4'])
|
||||
})
|
||||
})
|
||||
|
||||
it('removes the object when the multipart upload fails', async () => {
|
||||
await withTempFile(Buffer.alloc(16), async (path) => {
|
||||
const removed: string[] = []
|
||||
await expect(
|
||||
createPrivateMediaFromFile({
|
||||
inputPath: path,
|
||||
thresholdBytes: 0,
|
||||
create: async () => ({ id: 9, filename: 'falha.mp4' }),
|
||||
update: async () => undefined,
|
||||
uploadLarge: async () => {
|
||||
throw new Error('S3 indisponível.')
|
||||
},
|
||||
removeLarge: async ({ filename }) => {
|
||||
removed.push(filename)
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('S3 indisponível.')
|
||||
expect(removed).toEqual(['falha.mp4'])
|
||||
})
|
||||
})
|
||||
|
||||
it('fails closed when the placeholder row has no filename', async () => {
|
||||
await withTempFile(Buffer.alloc(16), async (path) => {
|
||||
let uploads = 0
|
||||
await expect(
|
||||
createPrivateMediaFromFile({
|
||||
inputPath: path,
|
||||
thresholdBytes: 0,
|
||||
create: async () => ({ id: 9, filename: null }),
|
||||
update: async () => undefined,
|
||||
uploadLarge: async () => {
|
||||
uploads += 1
|
||||
throw new Error('não deveria subir')
|
||||
},
|
||||
removeLarge: async () => undefined,
|
||||
}),
|
||||
).rejects.toThrow('filename')
|
||||
expect(uploads).toBe(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -8,15 +8,11 @@ import {
|
|||
formatRecordingFileSize,
|
||||
isRecordingStatus,
|
||||
RECORDING_FAILURE_INTERRUPTED,
|
||||
RECORDING_MAX_BYTES,
|
||||
RECORDING_MAX_SIZE_LABEL,
|
||||
RECORDING_STATUSES,
|
||||
RECORDING_STEPS,
|
||||
recordingFailureMessage,
|
||||
recordingFileTypeAllowed,
|
||||
recordingStatusLabels,
|
||||
recordingTooLargeMessage,
|
||||
recordingUploadTooLargeMessage,
|
||||
toRecordingViewModel,
|
||||
} from '@/lib/recording'
|
||||
import {
|
||||
|
|
@ -78,13 +74,9 @@ describe('recording vocabulary (C199)', () => {
|
|||
expect(recordingFileTypeAllowed('image/png')).toBe(false)
|
||||
})
|
||||
|
||||
it('formats sizes in pt-BR and names the limit', () => {
|
||||
expect(formatRecordingFileSize(4.6 * 1024 ** 3)).toContain('GB')
|
||||
expect(formatRecordingFileSize(812 * 1024 ** 2)).toBe('812 MB')
|
||||
expect(recordingTooLargeMessage(4.6 * 1024 ** 3)).toContain('4,6 GB')
|
||||
expect(recordingTooLargeMessage(4.6 * 1024 ** 3)).toContain(RECORDING_MAX_SIZE_LABEL)
|
||||
expect(recordingUploadTooLargeMessage).toContain(RECORDING_MAX_SIZE_LABEL)
|
||||
expect(RECORDING_MAX_BYTES).toBe(4 * 1024 ** 3)
|
||||
it('formats sizes in pt-BR without imposing a ceiling', () => {
|
||||
expect(formatRecordingFileSize(4.6 * 1024 ** 3)).toBe('4,6 GB')
|
||||
expect(formatRecordingFileSize(24 * 1024 ** 3)).toBe('24 GB')
|
||||
})
|
||||
|
||||
it('formats the byte range below 1 KB honestly', () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue