license import { ENGINE } from '../engine' ; import { env } from '../environment' ; import { FromPixels , FromPixelsAttrs , FromPixelsInputs } from '../kernel_names' ; import { getKernel , NamedAttrMap } from '../kernel_registry' ; import { Tensor , Tensor2D , Tensor3D } from '../tensor' ; import { NamedTensorMap } from '../tensor_types' ; import { convertToTensor } from '../tensor_util_env' ; import { PixelData , TensorLike } from '../types' ; import { cast } from './cast' ; import { op } from './operation' ; import { tensor3d } from './tensor3d' ; let fromPixels2DContext ; param pixels param numChannels returns doc function fromPixels_ ( pixels , numChannels = 3 ) { if ( numChannels > 4 ) { throw new Error ( 'Cannot construct Tensor with more than 4 channels from pixels.' ) ; } if ( pixels == null ) { throw new Error ( 'pixels passed to tf.browser.fromPixels() can not be null' ) ; } let isPixelData = false ; let isImageData = false ; let isVideo = false ; let isImage = false ; let isCanvasLike = false ; let isImageBitmap = false ; if ( ( pixels as PixelData ) . data instanceof Uint8Array ) { isPixelData = true ; } else if ( typeof ( ImageData ) !== 'undefined' && pixels instanceof ImageData ) { isImageData = true ; } else if ( typeof ( HTMLVideoElement ) !== 'undefined' && pixels instanceof HTMLVideoElement ) { isVideo = true ; } else if ( typeof ( HTMLImageElement ) !== 'undefined' && pixels instanceof HTMLImageElement ) { isImage = true ; } else if ( ( pixels as any ) . getContext != null ) { isCanvasLike = true ; } else if ( typeof ( ImageBitmap ) !== 'undefined' && pixels instanceof ImageBitmap ) { isImageBitmap = true ; } else { throw new Error ( 'pixels passed to tf.browser.fromPixels() must be either an ' + `HTMLVideoElement, HTMLImageElement, HTMLCanvasElement, ImageData ` + `in browser, or OffscreenCanvas, ImageData in webworker` + ` or {data: Uint32Array, width: number, height: number}, ` + `but was ${ ( pixels as { } ) . constructor . name }` ) ; } if ( isVideo ) { const HAVE_CURRENT_DATA_READY_STATE = 2 ; if ( isVideo && ( pixels as HTMLVideoElement ) . readyState < HAVE_CURRENT_DATA_READY_STATE ) { throw new Error ( 'The video element has not loaded data yet. Please wait for ' + '`loadeddata` event on the <video> element.' ) ; } } const kernel = getKernel ( FromPixels , ENGINE . backendName ) ; if ( kernel != null ) { const inputs = { pixels } ; const attrs = { numChannels } ; return ENGINE . runKernel ( FromPixels , inputs as { } as NamedTensorMap , attrs as { } as NamedAttrMap ) ; } const [ width , height ] = isVideo ? [ ( pixels as HTMLVideoElement ) . videoWidth , ( pixels as HTMLVideoElement ) . videoHeight ] : [ pixels . width , pixels . height ] ; let vals ; if ( isCanvasLike ) { vals = ( pixels as any ) . getContext ( '2d' ) . getImageData ( 0 , 0 , width , height ) . data ; } else if ( isImageData || isPixelData ) { vals = ( pixels as PixelData | ImageData ) . data ; } else if ( isImage || isVideo || isImageBitmap ) { if ( fromPixels2DContext == null ) { if ( typeof document === 'undefined' ) { if ( typeof OffscreenCanvas !== 'undefined' && typeof OffscreenCanvasRenderingContext2D !== 'undefined' ) { fromPixels2DContext = new OffscreenCanvas ( 1 , 1 ) . getContext ( '2d' ) ; } else { throw new Error ( 'Cannot parse input in current context. ' + 'Reason: OffscreenCanvas Context2D rendering is not supported.' ) ; } } else { fromPixels2DContext = document . createElement ( 'canvas' ) . getContext ( '2d' ) ; } } fromPixels2DContext . canvas . width = width ; fromPixels2DContext . canvas . height = height ; fromPixels2DContext . drawImage ( pixels as HTMLVideoElement , 0 , 0 , width , height ) ; vals = fromPixels2DContext . getImageData ( 0 , 0 , width , height ) . data ; } let values ; if ( numChannels === 4 ) { values = new Int32Array ( vals ) ; } else { const numPixels = width * height ; values = new Int32Array ( numPixels * numChannels ) ; for ( let i = 0 ; i < numPixels ; i ++ ) { for ( let channel = 0 ; channel < numChannels ; ++ channel ) { values [ i * numChannels + channel ] = vals [ i * 4 + channel ] ; } } } const outShape = [ height , width , numChannels ] ; return tensor3d ( values , outShape , 'int32' ) ; } function isPixelData ( pixels ) { return ( pixels != null ) && ( ( pixels as PixelData ) . data instanceof Uint8Array ) ; } function isImageBitmapFullySupported ( ) { return typeof window !== 'undefined' && typeof ( ImageBitmap ) !== 'undefined' && window . hasOwnProperty ( 'createImageBitmap' ) ; } function isNonEmptyPixels ( pixels ) { return pixels != null && pixels . width !== 0 && pixels . height !== 0 ; } function canWrapPixelsToImageBitmap ( pixels ) { return isImageBitmapFullySupported ( ) && ! ( pixels instanceof ImageBitmap ) && isNonEmptyPixels ( pixels ) && ! isPixelData ( pixels ) ; } param pixels param numChannels doc export async function fromPixelsAsync ( pixels , numChannels = 3 ) { let inputs = null ; if ( env ( ) . getBool ( 'WRAP_TO_IMAGEBITMAP' ) && canWrapPixelsToImageBitmap ( pixels ) ) { let imageBitmap ; try { imageBitmap = await ( createImageBitmap as any ) ( pixels as ImageBitmapSource , { premultiplyAlpha : 'none' } ) ; } catch ( e ) { imageBitmap = null ; } if ( imageBitmap != null && imageBitmap . width === pixels . width && imageBitmap . height === pixels . height ) { inputs = imageBitmap ; } else { inputs = pixels ; } } else { inputs = pixels ; } return fromPixels_ ( inputs , numChannels ) ; } param img param canvas doc export async function toPixels ( img , canvas ? ) { let $img = convertToTensor ( img , 'img' , 'toPixels' ) ; if ( ! ( img instanceof Tensor ) ) { const originalImgTensor = $img ; $img = cast ( originalImgTensor , 'int32' ) ; originalImgTensor . dispose ( ) ; } if ( $img . rank !== 2 && $img . rank !== 3 ) { throw new Error ( `toPixels only supports rank 2 or 3 tensors, got rank ${ $img . rank }.` ) ; } const [ height , width ] = $img . shape . slice ( 0 , 2 ) ; const depth = $img . rank === 2 ? 1 : $img . shape [ 2 ] ; if ( depth > 4 || depth === 2 ) { throw new Error ( `toPixels only supports depth of size ` + `1, 3 or 4 but got ${ depth }` ) ; } if ( $img . dtype !== 'float32' && $img . dtype !== 'int32' ) { throw new Error ( `Unsupported type for toPixels: ${ $img . dtype }.` + ` Please use float32 or int32 tensors.` ) ; } const data = await $img . data ( ) ; const multiplier = $img . dtype === 'float32' ? 255 : 1 ; const bytes = new Uint8ClampedArray ( width * height * 4 ) ; for ( let i = 0 ; i < height * width ; ++ i ) { const rgba = [ 0 , 0 , 0 , 255 ] ; for ( let d = 0 ; d < depth ; d ++ ) { const value = data [ i * depth + d ] ; if ( $img . dtype === 'float32' ) { if ( value < 0 || value > 1 ) { throw new Error ( `Tensor values for a float32 Tensor must be in the ` + `range [0 - 1] but encountered ${ value }.` ) ; } } else if ( $img . dtype === 'int32' ) { if ( value < 0 || value > 255 ) { throw new Error ( `Tensor values for a int32 Tensor must be in the ` + `range [0 - 255] but encountered ${ value }.` ) ; } } if ( depth === 1 ) { rgba [ 0 ] = value * multiplier ; rgba [ 1 ] = value * multiplier ; rgba [ 2 ] = value * multiplier ; } else { rgba [ d ] = value * multiplier ; } } const j = i * 4 ; bytes [ j + 0 ] = Math . round ( rgba [ 0 ] ) ; bytes [ j + 1 ] = Math . round ( rgba [ 1 ] ) ; bytes [ j + 2 ] = Math . round ( rgba [ 2 ] ) ; bytes [ j + 3 ] = Math . round ( rgba [ 3 ] ) ; } if ( canvas != null ) { canvas . width = width ; canvas . height = height ; const ctx = canvas . getContext ( '2d' ) ; const imageData = new ImageData ( bytes , width , height ) ; ctx . putImageData ( imageData , 0 , 0 ) ; } if ( $img !== img ) { $img . dispose ( ) ; } return bytes ; } export const fromPixels = op ( { fromPixels_ } ) ;