Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 | /** * Perspective Transform utilities using OpenCV.js * * Provides functions to rectify a quadrilateral region from a video frame * into a rectangle, correcting for camera perspective distortion. */ import type { QuadCorners } from '@/types/vision' // OpenCV.js types (minimal subset we need) declare global { interface Window { cv: { Mat: new (rows?: number, cols?: number, type?: number) => CvMat matFromImageData: (imageData: ImageData) => CvMat matFromArray: (rows: number, cols: number, type: number, data: number[]) => CvMat CV_8UC4: number CV_32FC1: number CV_32FC2: number getPerspectiveTransform: (src: CvMat, dst: CvMat) => CvMat warpPerspective: ( src: CvMat, dst: CvMat, M: CvMat, dsize: { width: number; height: number }, flags?: number, borderMode?: number, borderValue?: unknown ) => void cvtColor: (src: CvMat, dst: CvMat, code: number) => void COLOR_RGBA2RGB: number COLOR_RGB2RGBA: number INTER_LINEAR: number BORDER_CONSTANT: number onRuntimeInitialized?: () => void } } interface CvMat { rows: number cols: number data: Uint8Array data32F: Float32Array delete: () => void } } let opencvLoaded = false let opencvLoadPromise: Promise<void> | null = null /** * Load OpenCV.js dynamically */ export async function loadOpenCV(): Promise<void> { if (opencvLoaded) return if (opencvLoadPromise) { return opencvLoadPromise } opencvLoadPromise = new Promise<void>((resolve, reject) => { // Check if already loaded if (typeof window !== 'undefined' && window.cv && window.cv.Mat) { opencvLoaded = true resolve() return } // Load the script const script = document.createElement('script') script.src = '/opencv.js' script.async = true script.onload = () => { // OpenCV.js uses a callback when ready if (window.cv && window.cv.onRuntimeInitialized !== undefined) { window.cv.onRuntimeInitialized = () => { opencvLoaded = true resolve() } } else { // Fallback: poll for cv.Mat availability const checkReady = setInterval(() => { if (window.cv && window.cv.Mat) { clearInterval(checkReady) opencvLoaded = true resolve() } }, 100) // Timeout after 10 seconds setTimeout(() => { clearInterval(checkReady) if (!opencvLoaded) { reject(new Error('OpenCV.js failed to initialize')) } }, 10000) } } script.onerror = () => { reject(new Error('Failed to load OpenCV.js')) } document.head.appendChild(script) }) return opencvLoadPromise } /** * Check if OpenCV is loaded and ready */ export function isOpenCVReady(): boolean { return opencvLoaded && typeof window !== 'undefined' && !!window.cv?.Mat } export interface RectifyOptions { /** Output width (default: calculated from quad aspect ratio) */ outputWidth?: number /** Output height (default: calculated from quad aspect ratio) */ outputHeight?: number /** * Apply 180° rotation to output (default: true for backwards compatibility) * Set to false for phone cameras where no rotation is needed */ rotate180?: boolean /** * Number of abacus columns (default: undefined) * When provided, uses a fixed aspect ratio based on column count: * - 5-column abacus = 1:1 aspect ratio (width:height) * - 4-column abacus = 4:5 aspect ratio (taller than wide) * Formula: width:height = columnCount:5 */ columnCount?: number } /** * Rectify a quadrilateral region from video to a rectangle * * @param video - Source video element * @param corners - Quadrilateral corners in video coordinates * @param canvas - Destination canvas for rectified output * @param options - Output size options */ export function rectifyQuadrilateral( video: HTMLVideoElement, corners: QuadCorners, canvas: HTMLCanvasElement, options: RectifyOptions = {} ): boolean { if (!isOpenCVReady()) { console.warn('[perspectiveTransform] OpenCV not ready') return false } const cv = window.cv const rotate180 = options.rotate180 ?? true // Default true for backwards compatibility // Calculate default output dimensions based on quad size const topWidth = Math.hypot( corners.topRight.x - corners.topLeft.x, corners.topRight.y - corners.topLeft.y ) const bottomWidth = Math.hypot( corners.bottomRight.x - corners.bottomLeft.x, corners.bottomRight.y - corners.bottomLeft.y ) const leftHeight = Math.hypot( corners.bottomLeft.x - corners.topLeft.x, corners.bottomLeft.y - corners.topLeft.y ) const rightHeight = Math.hypot( corners.bottomRight.x - corners.topRight.x, corners.bottomRight.y - corners.topRight.y ) const avgWidth = (topWidth + bottomWidth) / 2 const avgHeight = (leftHeight + rightHeight) / 2 // Calculate output dimensions let outputWidth: number let outputHeight: number if (options.columnCount !== undefined) { // Use FIXED dimensions based on column count - completely independent of detected quad // 5 columns = 1:1 (square) // 4 columns = 4:5 (narrower, since fewer columns) // Width scales with column count, height is fixed // // Use a fixed base size of 400 pixels for the height (5 units) const BASE_SIZE = 400 outputWidth = options.outputWidth ?? Math.round((BASE_SIZE * options.columnCount) / 5) outputHeight = options.outputHeight ?? BASE_SIZE } else { // Fall back to detected quad dimensions outputWidth = options.outputWidth ?? Math.round(avgWidth) outputHeight = options.outputHeight ?? Math.round(avgHeight) } // Set canvas size canvas.width = outputWidth canvas.height = outputHeight // Create temporary canvas to capture video frame const tempCanvas = document.createElement('canvas') tempCanvas.width = video.videoWidth tempCanvas.height = video.videoHeight const tempCtx = tempCanvas.getContext('2d') if (!tempCtx) return false // Draw video frame tempCtx.drawImage(video, 0, 0) const imageData = tempCtx.getImageData(0, 0, tempCanvas.width, tempCanvas.height) // Create OpenCV matrices let srcMat: CvMat | null = null let dstMat: CvMat | null = null let srcPoints: CvMat | null = null let dstPoints: CvMat | null = null let transformMatrix: CvMat | null = null try { // Source image srcMat = cv.matFromImageData(imageData) // Source points mapping // When rotate180=true (Desk View): swap diagonally so output is 180° rotated // When rotate180=false (normal phone): direct mapping, no rotation const srcPointsData = rotate180 ? [ // Desk View: swap diagonally for camera pointing down corners.bottomRight.x, corners.bottomRight.y, // screen BR → output TL corners.bottomLeft.x, corners.bottomLeft.y, // screen BL → output TR corners.topLeft.x, corners.topLeft.y, // screen TL → output BR corners.topRight.x, corners.topRight.y, // screen TR → output BL ] : [ // Phone camera: direct mapping, no rotation corners.topLeft.x, corners.topLeft.y, // TL → output TL corners.topRight.x, corners.topRight.y, // TR → output TR corners.bottomRight.x, corners.bottomRight.y, // BR → output BR corners.bottomLeft.x, corners.bottomLeft.y, // BL → output BL ] srcPoints = cv.matFromArray(4, 1, cv.CV_32FC2, srcPointsData) // Destination points (rectangle corners) - map to standard rectangle // Order: TL, TR, BR, BL dstPoints = cv.matFromArray(4, 1, cv.CV_32FC2, [ 0, 0, // TL → top-left of output outputWidth, 0, // TR → top-right of output outputWidth, outputHeight, // BR → bottom-right of output 0, outputHeight, // BL → bottom-left of output ]) // Calculate perspective transform matrix transformMatrix = cv.getPerspectiveTransform(srcPoints, dstPoints) // Create output matrix dstMat = new cv.Mat(outputHeight, outputWidth, cv.CV_8UC4) // Apply perspective warp cv.warpPerspective( srcMat, dstMat, transformMatrix, { width: outputWidth, height: outputHeight }, cv.INTER_LINEAR, cv.BORDER_CONSTANT ) // Copy result to canvas const outputCtx = canvas.getContext('2d') if (!outputCtx) return false const outputData = new ImageData(new Uint8ClampedArray(dstMat.data), outputWidth, outputHeight) outputCtx.putImageData(outputData, 0, 0) return true } catch (err) { console.error('[perspectiveTransform] Error:', err) return false } finally { // Clean up OpenCV matrices srcMat?.delete() dstMat?.delete() srcPoints?.delete() dstPoints?.delete() transformMatrix?.delete() } } export interface RectifyToBase64Options extends RectifyOptions { /** JPEG quality (0-1, default 0.8) */ jpegQuality?: number } /** * Rectify a quadrilateral region from video to a base64-encoded JPEG * * This is the function to use for remote camera streaming - it returns * a base64 string ready to send over the network. * * @param video - Source video element * @param corners - Quadrilateral corners in video coordinates * @param options - Output size and quality options * @returns Base64-encoded JPEG (without data URL prefix), or null on failure */ export function rectifyQuadrilateralToBase64( video: HTMLVideoElement, corners: QuadCorners, options: RectifyToBase64Options = {} ): string | null { const { jpegQuality = 0.8, ...rectifyOptions } = options // Create a temporary canvas const canvas = document.createElement('canvas') // Use the existing rectifyQuadrilateral function const success = rectifyQuadrilateral(video, corners, canvas, rectifyOptions) if (!success) { return null } // Convert to JPEG and return base64 (without data URL prefix) const dataUrl = canvas.toDataURL('image/jpeg', jpegQuality) return dataUrl.split(',')[1] } /** * Create a rectified frame processor that continuously updates a canvas * * @param video - Source video element * @param corners - Quadrilateral corners * @param canvas - Output canvas * @param options - Rectify options * @returns Stop function */ export function createRectifiedFrameLoop( video: HTMLVideoElement, corners: QuadCorners, canvas: HTMLCanvasElement, options: RectifyOptions = {} ): () => void { let running = true let animationId: number | null = null const processFrame = () => { if (!running) return if (video.readyState >= 2) { // HAVE_CURRENT_DATA rectifyQuadrilateral(video, corners, canvas, options) } animationId = requestAnimationFrame(processFrame) } processFrame() return () => { running = false if (animationId !== null) { cancelAnimationFrame(animationId) } } } |