mirror of
https://github.com/fsolla/teqo.git
synced 2026-11-17 21:03:06 +00:00
O recorte do card 'Time de você' falhava em navegadores endurecidos (IronFox/Tor desligam WebAssembly e WebGL por padrão) com a copy errada de 'tente outra foto'; o motor MediaPipe exige ambos (o grafo pede kGpuService mesmo com delegate CPU). Agora o adapter sonda WASM/WebGL antes do download e devolve 'unsupported' com mensagem acionável; a entrada de inferência é capada em 1600 e a foto passa pelo loadCardPhoto (foto de 61MP que falhava agora conclui). O enquadramento automático volta ao S15 (bbox cobre a janela): o S18 mirava a caixa do rosto em 103px e deixava o busto inteiro pequeno na janela. Saem o detector de rosto (asset removido), frameCardPhotoOnFace, a referência 103px, o piso 0.2 e a âncora S20 (inerte sob o cover). S16/S17 intactos.
394 lines
14 KiB
TypeScript
394 lines
14 KiB
TypeScript
/**
|
||
* S15 — on-device background removal for the `Time de você` card. The visitor's
|
||
* photo never leaves the browser: the model + wasm are fetched from our own
|
||
* `/cards` static assets and inference runs locally. Kept out of `src/lib`
|
||
* because it touches DOM/worker APIs; the pure framing math stays in
|
||
* `cardPhotoTransform`.
|
||
*
|
||
* Engine: MediaPipe `@mediapipe/tasks-vision` ImageSegmenter with the Apache-2.0
|
||
* `selfie_segmenter.tflite` (float16, 256×256) — no COOP/COEP, single-threaded
|
||
* wasm, lazily imported only when the visitor picks the team model. The
|
||
* `@imgly` packages stay out of the bundle (AGPL + ~40 MB on mobile).
|
||
*
|
||
* Hardened browsers (IronFox, Tor-like builds) disable WebGL and/or WebAssembly
|
||
* by default, which breaks the engine before any photo is processed. The engine
|
||
* now probes both up front (a truthful retry message instead of a misleading
|
||
* "try another photo") — MediaPipe's vision graph needs a WebGL context even
|
||
* with the CPU delegate, so there is no software fallback. The inference input
|
||
* is capped before it reaches the runtime so a full-resolution phone photo
|
||
* never becomes a giant GPU texture/allocation.
|
||
*/
|
||
|
||
import type { ImageSegmenter, MPMask } from '@mediapipe/tasks-vision'
|
||
|
||
import { loadCardPhoto } from '@/components/cards/cardCanvas'
|
||
import { CARD_CUTOUT_ALPHA_THRESHOLD, type CardAlphaBbox } from '@/lib/cardPhotoTransform'
|
||
|
||
declare global {
|
||
interface Window {
|
||
/** Test seam: the e2e build stubs the engine through this flag. */
|
||
__cardsCutoutStub?: 'ok' | 'slow' | 'error'
|
||
}
|
||
}
|
||
|
||
const WASM_DIR = '/cards/mediapipe/wasm'
|
||
const MODEL_URL = '/cards/selfie_segmenter.tflite'
|
||
const CUTOUT_MAX_EDGE = 1600
|
||
const STUB_ENABLED = process.env.NEXT_PUBLIC_CARDS_CUTOUT_STUB === '1'
|
||
const STUB_DELAY_MS = 1500
|
||
const STUB_PROGRESS = 0.36
|
||
/** Upscale target so the synthetic bbox passes the real cutout checks. */
|
||
const STUB_MIN_EDGE = 400
|
||
|
||
export type CardCutoutProgress = {
|
||
phase: 'download' | 'process'
|
||
/** 0..1 — real bytes while downloading, a coarse step while processing. */
|
||
ratio: number
|
||
}
|
||
|
||
type CardCutoutFailureReason = 'engine' | 'empty' | 'unsupported'
|
||
|
||
export type CardCutoutResult =
|
||
| {
|
||
ok: true
|
||
canvas: HTMLCanvasElement
|
||
width: number
|
||
height: number
|
||
bbox: CardAlphaBbox
|
||
}
|
||
| { ok: false; reason: CardCutoutFailureReason }
|
||
|
||
/**
|
||
* Removes the background of `file` on this device. Resolves with the cutout
|
||
* canvas + its alpha bbox, or fails with `engine` (decode/inference/asset
|
||
* failure), `empty` (blank segmentation) or `unsupported` (the browser blocks
|
||
* WebAssembly/WebGL — the local engine cannot run at all) — all recoverable in
|
||
* the composer.
|
||
*/
|
||
export const removeCardPhotoBackground = async (
|
||
file: File,
|
||
onProgress: (progress: CardCutoutProgress) => void,
|
||
): Promise<CardCutoutResult> => {
|
||
const report = createProgressReporter(onProgress)
|
||
|
||
if (STUB_ENABLED) {
|
||
try {
|
||
return await stubCutout(file, report)
|
||
} catch {
|
||
return { ok: false, reason: 'engine' }
|
||
}
|
||
}
|
||
|
||
if (!supportsWebAssembly() || !supportsWebgl()) return { ok: false, reason: 'unsupported' }
|
||
|
||
try {
|
||
return await engineCutout(file, report)
|
||
} catch {
|
||
return { ok: false, reason: 'engine' }
|
||
}
|
||
}
|
||
|
||
/**
|
||
* WebAssembly availability probe: hardened browsers can hide the global
|
||
* entirely (`javascript.options.wasm` off) — the engine cannot run without it,
|
||
* so the composer must say so up front instead of blaming the photo. Exported
|
||
* for unit tests.
|
||
*/
|
||
export const supportsWebAssembly = (): boolean => {
|
||
try {
|
||
// Minimal valid module header — cheap and only true where compiling works.
|
||
return WebAssembly.validate(new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0]))
|
||
} catch {
|
||
return false
|
||
}
|
||
}
|
||
|
||
/**
|
||
* WebGL availability probe: MediaPipe's vision graph requests a WebGL2 context
|
||
* (falling back to WebGL1) for its image pipeline even when the inference
|
||
* delegate is CPU, so without it the engine always fails. Probed before the
|
||
* ~12 MB download. Exported for unit tests.
|
||
*/
|
||
export const supportsWebgl = (): boolean => {
|
||
if (typeof document === 'undefined') return false
|
||
|
||
try {
|
||
const canvas = document.createElement('canvas')
|
||
const context = canvas.getContext('webgl2') ?? canvas.getContext('webgl')
|
||
if (!context) return false
|
||
|
||
// Release the probe context right away: repeated retries must never
|
||
// accumulate toward the browser's live-context cap.
|
||
context.getExtension('WEBGL_lose_context')?.loseContext()
|
||
return true
|
||
} catch {
|
||
return false
|
||
}
|
||
}
|
||
|
||
const PROGRESS_MIN_INTERVAL_MS = 100
|
||
|
||
/**
|
||
* Streaming a ~12 MB wasm fires a progress event per chunk; without throttle
|
||
* that is hundreds of React updates in one microtask chain (the dev build
|
||
* reports "Maximum update depth exceeded"). One report per 100 ms — always
|
||
* emitting phase changes and completion — keeps the bar honest and the UI calm.
|
||
*/
|
||
const createProgressReporter = (onProgress: (progress: CardCutoutProgress) => void) => {
|
||
let lastAt = 0
|
||
let lastPhase: CardCutoutProgress['phase'] | null = null
|
||
|
||
return (progress: CardCutoutProgress) => {
|
||
const now = Date.now()
|
||
const phaseChanged = progress.phase !== lastPhase
|
||
const finished = progress.ratio >= 1
|
||
if (!phaseChanged && !finished && now - lastAt < PROGRESS_MIN_INTERVAL_MS) return
|
||
|
||
lastAt = now
|
||
lastPhase = progress.phase
|
||
onProgress(progress)
|
||
}
|
||
}
|
||
|
||
let segmenterPromise: Promise<ImageSegmenter> | null = null
|
||
|
||
const loadSegmenter = (onProgress: (progress: CardCutoutProgress) => void) => {
|
||
if (!segmenterPromise) {
|
||
segmenterPromise = createSegmenter(onProgress).catch((error: unknown) => {
|
||
segmenterPromise = null
|
||
throw error
|
||
})
|
||
}
|
||
|
||
return segmenterPromise
|
||
}
|
||
|
||
const createSegmenter = async (onProgress: (progress: CardCutoutProgress) => void) => {
|
||
const wasmFiles = supportsWasmSimd()
|
||
? ['vision_wasm_internal.js', 'vision_wasm_internal.wasm']
|
||
: ['vision_wasm_nosimd_internal.js', 'vision_wasm_nosimd_internal.wasm']
|
||
await prefetchAssets([...wasmFiles.map((name) => `${WASM_DIR}/${name}`), MODEL_URL], onProgress)
|
||
|
||
const vision = await import('@mediapipe/tasks-vision')
|
||
const fileset = await vision.FilesetResolver.forVisionTasks(WASM_DIR)
|
||
const modelAssetBuffer = await fetchModelBuffer(MODEL_URL, 'card-cutout-model')
|
||
|
||
return vision.ImageSegmenter.createFromOptions(fileset, {
|
||
baseOptions: { modelAssetBuffer },
|
||
runningMode: 'IMAGE',
|
||
outputConfidenceMasks: true,
|
||
outputCategoryMask: false,
|
||
})
|
||
}
|
||
|
||
const fetchModelBuffer = async (url: string, errorPrefix: string) => {
|
||
const response = await fetch(url)
|
||
if (!response.ok) throw new Error(`${errorPrefix}:${response.status}`)
|
||
|
||
return new Uint8Array(await response.arrayBuffer())
|
||
}
|
||
|
||
/**
|
||
* Streams each asset once so the progress bar reflects real bytes — and warms
|
||
* the HTTP cache for the runtimes that later fetch the same URLs themselves.
|
||
*/
|
||
const prefetchAssets = async (
|
||
urls: string[],
|
||
onProgress: (progress: CardCutoutProgress) => void,
|
||
) => {
|
||
let loadedFiles = 0
|
||
|
||
for (const url of urls) {
|
||
const response = await fetch(url)
|
||
if (!response.ok) throw new Error(`card-cutout-asset:${response.status}`)
|
||
const size = Number(response.headers.get('content-length')) || 0
|
||
|
||
if (!response.body) {
|
||
await response.arrayBuffer()
|
||
loadedFiles += 1
|
||
onProgress({ phase: 'download', ratio: Math.min(1, loadedFiles / urls.length) })
|
||
continue
|
||
}
|
||
|
||
const reader = response.body.getReader()
|
||
let received = 0
|
||
for (;;) {
|
||
const { done, value } = await reader.read()
|
||
if (done) break
|
||
received += value?.length ?? 0
|
||
const fileRatio = size > 0 ? Math.min(1, received / size) : 0.5
|
||
onProgress({ phase: 'download', ratio: Math.min(1, (loadedFiles + fileRatio) / urls.length) })
|
||
}
|
||
loadedFiles += 1
|
||
onProgress({ phase: 'download', ratio: Math.min(1, loadedFiles / urls.length) })
|
||
}
|
||
}
|
||
|
||
const engineCutout = async (
|
||
file: File,
|
||
onProgress: (progress: CardCutoutProgress) => void,
|
||
): Promise<CardCutoutResult> => {
|
||
const segmenter = await loadSegmenter(onProgress)
|
||
onProgress({ phase: 'process', ratio: 0.5 })
|
||
|
||
// The photo is decoded through the shared card-photo loader (`<img>`, EXIF
|
||
// orientation applied, iOS-subsampled decodes) and downscaled to the cap
|
||
// *before* it reaches the runtime: full-resolution phone photos otherwise
|
||
// become giant GPU textures/allocs and can blank the segmentation.
|
||
const photo = await loadCardPhoto(file)
|
||
const { width, height } = cappedSize(photo.naturalWidth, photo.naturalHeight)
|
||
const canvas = document.createElement('canvas')
|
||
canvas.width = width
|
||
canvas.height = height
|
||
const ctx = canvas.getContext('2d')
|
||
if (!ctx) throw new Error('card-cutout-context')
|
||
ctx.imageSmoothingEnabled = true
|
||
ctx.imageSmoothingQuality = 'high'
|
||
ctx.drawImage(photo, 0, 0, width, height)
|
||
|
||
// Let the processing state paint before the synchronous inference.
|
||
await new Promise<void>((resolve) => requestAnimationFrame(() => resolve()))
|
||
|
||
const result = segmenter.segment(canvas)
|
||
let mask: MPMask | null = null
|
||
try {
|
||
mask = result.confidenceMasks?.[0] ?? null
|
||
if (!mask) return { ok: false, reason: 'engine' }
|
||
|
||
applyMask(canvas, mask)
|
||
const bbox = readAlphaBbox(canvas)
|
||
if (!bbox) return { ok: false, reason: 'empty' }
|
||
|
||
onProgress({ phase: 'process', ratio: 1 })
|
||
return { ok: true, canvas, width, height, bbox }
|
||
} finally {
|
||
mask?.close()
|
||
result.close()
|
||
}
|
||
}
|
||
|
||
const applyMask = (canvas: HTMLCanvasElement, mask: MPMask): void => {
|
||
const maskCanvas = document.createElement('canvas')
|
||
maskCanvas.width = mask.width
|
||
maskCanvas.height = mask.height
|
||
const maskCtx = maskCanvas.getContext('2d')
|
||
if (!maskCtx) throw new Error('card-cutout-mask-context')
|
||
|
||
const confidence = mask.getAsFloat32Array()
|
||
const maskImage = maskCtx.createImageData(mask.width, mask.height)
|
||
for (let index = 0, alpha = 3; index < confidence.length; index += 1, alpha += 4) {
|
||
const value = confidence[index] ?? 0
|
||
maskImage.data[alpha] = value <= 0 ? 0 : value >= 1 ? 255 : Math.round(value * 255)
|
||
}
|
||
maskCtx.putImageData(maskImage, 0, 0)
|
||
|
||
const ctx = canvas.getContext('2d')
|
||
if (!ctx) throw new Error('card-cutout-context')
|
||
|
||
ctx.globalCompositeOperation = 'destination-in'
|
||
ctx.imageSmoothingEnabled = true
|
||
ctx.imageSmoothingQuality = 'high'
|
||
ctx.drawImage(maskCanvas, 0, 0, canvas.width, canvas.height)
|
||
ctx.globalCompositeOperation = 'source-over'
|
||
}
|
||
|
||
const readAlphaBbox = (canvas: HTMLCanvasElement): CardAlphaBbox | null => {
|
||
const ctx = canvas.getContext('2d')
|
||
if (!ctx) return null
|
||
|
||
const { width, height } = canvas
|
||
const pixels = ctx.getImageData(0, 0, width, height).data
|
||
let minX = width
|
||
let minY = height
|
||
let maxX = -1
|
||
let maxY = -1
|
||
|
||
for (let y = 0; y < height; y += 1) {
|
||
const row = y * width * 4
|
||
for (let x = 0; x < width; x += 1) {
|
||
if (pixels[row + x * 4 + 3] > CARD_CUTOUT_ALPHA_THRESHOLD) {
|
||
if (x < minX) minX = x
|
||
if (x > maxX) maxX = x
|
||
if (y < minY) minY = y
|
||
if (y > maxY) maxY = y
|
||
}
|
||
}
|
||
}
|
||
|
||
if (maxX < minX || maxY < minY) return null
|
||
return { x: minX, y: minY, width: maxX - minX + 1, height: maxY - minY + 1 }
|
||
}
|
||
|
||
/** Exported for unit tests: the longest edge cap of the inference canvas. */
|
||
export const cappedSize = (width: number, height: number) => {
|
||
const longest = Math.max(width, height)
|
||
const scale = longest > CUTOUT_MAX_EDGE ? CUTOUT_MAX_EDGE / longest : 1
|
||
|
||
return {
|
||
width: Math.max(1, Math.round(width * scale)),
|
||
height: Math.max(1, Math.round(height * scale)),
|
||
}
|
||
}
|
||
|
||
const supportsWasmSimd = (): boolean => {
|
||
// Minimal wasm module using `v128.const` — validates only where SIMD exists.
|
||
const probe = new Uint8Array([
|
||
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7b, 0x03,
|
||
0x02, 0x01, 0x00, 0x0a, 0x0a, 0x01, 0x08, 0x00, 0x41, 0x00, 0xfd, 0x0f, 0x1a, 0x0b,
|
||
])
|
||
|
||
try {
|
||
return WebAssembly.validate(probe)
|
||
} catch {
|
||
return false
|
||
}
|
||
}
|
||
|
||
const readStubMode = (): 'ok' | 'slow' | 'error' => {
|
||
if (typeof window === 'undefined') return 'ok'
|
||
const mode = window.__cardsCutoutStub
|
||
return mode === 'slow' || mode === 'error' ? mode : 'ok'
|
||
}
|
||
|
||
/**
|
||
* Test seam (`NEXT_PUBLIC_CARDS_CUTOUT_STUB=1`, e2e builds only): a
|
||
* deterministic cutout with the same shape as the engine result. `slow` keeps
|
||
* the processing state visible; `error` exercises the retry path.
|
||
*/
|
||
const stubCutout = async (
|
||
file: File,
|
||
onProgress: (progress: CardCutoutProgress) => void,
|
||
): Promise<CardCutoutResult> => {
|
||
const mode = readStubMode()
|
||
onProgress({ phase: 'download', ratio: mode === 'slow' ? STUB_PROGRESS : 1 })
|
||
await new Promise((resolve) => setTimeout(resolve, mode === 'slow' ? STUB_DELAY_MS : 20))
|
||
if (mode === 'error') return { ok: false, reason: 'engine' }
|
||
onProgress({ phase: 'process', ratio: 0.8 })
|
||
|
||
const bitmap = await createImageBitmap(file)
|
||
const capped = cappedSize(bitmap.width, bitmap.height)
|
||
const stubScale = Math.max(1, STUB_MIN_EDGE / Math.max(capped.width, capped.height))
|
||
const width = Math.round(capped.width * stubScale)
|
||
const height = Math.round(capped.height * stubScale)
|
||
const canvas = document.createElement('canvas')
|
||
canvas.width = width
|
||
canvas.height = height
|
||
const ctx = canvas.getContext('2d')
|
||
if (!ctx) throw new Error('card-cutout-context')
|
||
|
||
ctx.drawImage(bitmap, 0, 0, width, height)
|
||
ctx.globalCompositeOperation = 'destination-in'
|
||
ctx.beginPath()
|
||
ctx.ellipse(width / 2, height / 2, width * 0.45, height * 0.45, 0, 0, Math.PI * 2)
|
||
ctx.fill()
|
||
ctx.globalCompositeOperation = 'source-over'
|
||
bitmap.close()
|
||
|
||
return {
|
||
ok: true,
|
||
canvas,
|
||
width,
|
||
height,
|
||
bbox: { x: 0, y: 0, width, height },
|
||
}
|
||
}
|