/********************************************************************************
 * Copyright (C) 2020 TypeFox and others.
 *
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Public License v. 2.0 which is available at
 * http://www.eclipse.org/legal/epl-2.0.
 *
 * This Source Code may also be made available under the following Secondary
 * Licenses when the conditions for such availability set forth in the Eclipse
 * Public License v. 2.0 are satisfied: GNU General Public License, version 2
 * with the GNU Classpath Exception which is available at
 * https://www.gnu.org/software/classpath/license.html.
 *
 * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 ********************************************************************************/ import * as iconv from 'iconv-lite' ; import { Buffer } from 'safer-buffer' ; import { injectable } from 'inversify' ; import { BinaryBuffer , BinaryBufferReadableStream , BinaryBufferReadable } from './buffer' ; import { UTF8 , UTF8_with_bom , UTF16be , UTF16le , UTF16be_BOM , UTF16le_BOM , UTF8_BOM } from './encodings' ; import { newWriteableStream , ReadableStream , Readable } from './stream' ; const ZERO_BYTE_DETECTION_BUFFER_MAX_LEN = 512 ; const NO_ENCODING_GUESS_MIN_BYTES = 512 ; const AUTO_ENCODING_GUESS_MIN_BYTES = 512 * 8 ; const AUTO_ENCODING_GUESS_MAX_BYTES = 512 * 128 ; const IGNORE_ENCODINGS = [ 'ascii' , 'utf-16' , 'utf-32' ] ; export interface ResourceEncoding { encoding : string ; hasBOM : boolean ; } export interface DetectedEncoding { encoding ? : string ; seemsBinary ? : boolean ; } export interface DecodeStreamOptions { guessEncoding ? : boolean ; minBytesRequiredForDetection ? : number ; overwriteEncoding ( detectedEncoding ) : Promise < string > ; } export interface DecodeStreamResult { stream : ReadableStream < string > ; detected : DetectedEncoding ; } @ injectable ( ) export class EncodingService { encode ( value , options ? ) { let encoding = options ?. encoding ; const addBOM = options ?. hasBOM ; encoding = this . toIconvEncoding ( encoding ) ; if ( encoding === UTF8 && ! addBOM ) { return BinaryBuffer . fromString ( value ) ; } const buffer = iconv . encode ( value , encoding , { addBOM } ) ; return BinaryBuffer . wrap ( buffer ) ; } decode ( value , encoding ? ) { const buffer = Buffer . from ( value . buffer ) ; encoding = this . toIconvEncoding ( encoding ) ; return iconv . decode ( buffer , encoding ) ; } exists ( encoding ) { encoding = this . toIconvEncoding ( encoding ) ; return iconv . encodingExists ( encoding ) ; } toIconvEncoding ( encoding ? ) { if ( encoding === UTF8_with_bom || ! encoding ) { return UTF8 ; } return encoding ; } async toResourceEncoding ( encoding , options ) { if ( encoding === UTF16be || encoding === UTF16le || encoding === UTF8_with_bom ) { return { encoding , hasBOM : true } ; } const overwriteEncoding = options ?. overwriteEncoding ; if ( ! overwriteEncoding && encoding === UTF8 ) { try { const buffer = await options . read ( UTF8_BOM . length ) ; if ( this . detectEncodingByBOMFromBuffer ( Buffer . from ( buffer ) , buffer . byteLength ) === UTF8_with_bom ) { return { encoding , hasBOM : true } ; } } catch ( error ) { } } return { encoding , hasBOM : false } ; } async detectEncoding ( data , autoGuessEncoding ? ) { const buffer = Buffer . from ( data . buffer ) ; const bytesRead = data . byteLength ; let encoding = this . detectEncodingByBOMFromBuffer ( buffer , bytesRead ) ; let seemsBinary = false ; if ( encoding !== UTF16be && encoding !== UTF16le && buffer ) { let couldBeUTF16LE = true ; let couldBeUTF16BE = true ; let containsZeroByte = false ; for ( let i = 0 ; i < bytesRead && i < ZERO_BYTE_DETECTION_BUFFER_MAX_LEN ; i ++ ) { const isEndian = ( i % 2 === 1 ) ; const isZeroByte = ( buffer . readUInt8 ( i ) === 0 ) ; if ( isZeroByte ) { containsZeroByte = true ; } if ( couldBeUTF16LE && ( isEndian && ! isZeroByte || ! isEndian && isZeroByte ) ) { couldBeUTF16LE = false ; } if ( couldBeUTF16BE && ( isEndian && isZeroByte || ! isEndian && ! isZeroByte ) ) { couldBeUTF16BE = false ; } if ( isZeroByte && ! couldBeUTF16LE && ! couldBeUTF16BE ) { break ; } } if ( containsZeroByte ) { if ( couldBeUTF16LE ) { encoding = UTF16le ; } else if ( couldBeUTF16BE ) { encoding = UTF16be ; } else { seemsBinary = true ; } } } if ( autoGuessEncoding && ! seemsBinary && ! encoding && buffer ) { const guessedEncoding = await this . guessEncodingByBuffer ( buffer . slice ( 0 , bytesRead ) ) ; return { seemsBinary : false , encoding : guessedEncoding } ; } return { seemsBinary , encoding } ; } protected detectEncodingByBOMFromBuffer ( buffer , bytesRead ) { if ( ! buffer || bytesRead < UTF16be_BOM . length ) { return undefined ; } const b0 = buffer . readUInt8 ( 0 ) ; const b1 = buffer . readUInt8 ( 1 ) ; if ( b0 === UTF16be_BOM [ 0 ] && b1 === UTF16be_BOM [ 1 ] ) { return UTF16be ; } if ( b0 === UTF16le_BOM [ 0 ] && b1 === UTF16le_BOM [ 1 ] ) { return UTF16le ; } if ( bytesRead < UTF8_BOM . length ) { return undefined ; } const b2 = buffer . readUInt8 ( 2 ) ; if ( b0 === UTF8_BOM [ 0 ] && b1 === UTF8_BOM [ 1 ] && b2 === UTF8_BOM [ 2 ] ) { return UTF8_with_bom ; } return undefined ; } protected async guessEncodingByBuffer ( buffer ) { const jschardet = await import ( 'jschardet' ) ; const guessed = jschardet . detect ( buffer . slice ( 0 , AUTO_ENCODING_GUESS_MAX_BYTES ) ) ; if ( ! guessed || ! guessed . encoding ) { return undefined ; } const enc = guessed . encoding . toLowerCase ( ) ; if ( 0 <= IGNORE_ENCODINGS . indexOf ( enc ) ) { return undefined ; } return this . toIconvEncoding ( guessed . encoding ) ; } decodeStream ( source , options ) { const minBytesRequiredForDetection = options . minBytesRequiredForDetection ?? options . guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES ; return new Promise < DecodeStreamResult > ( ( resolve , reject ) => { const target = newWriteableStream < string > ( strings => strings . join ( '' ) ) ; const bufferedChunks = [ ] ; let bytesBuffered = 0 ; let decoder = undefined ; const createDecoder = async ( ) => { try { const detected = await this . detectEncoding ( BinaryBuffer . concat ( bufferedChunks ) , options . guessEncoding ) ; detected . encoding = await options . overwriteEncoding ( detected . encoding ) ; decoder = iconv . getDecoder ( this . toIconvEncoding ( detected . encoding ) ) ; const decoded = decoder . write ( Buffer . from ( BinaryBuffer . concat ( bufferedChunks ) . buffer ) ) ; target . write ( decoded ) ; bufferedChunks . length = 0 ; bytesBuffered = 0 ; resolve ( { stream : target , detected } ) ; } catch ( error ) { reject ( error ) ; } } ; source . on ( 'error' , error => target . error ( error ) ) ; source . on ( 'data' , async ( chunk ) => { if ( decoder ) { target . write ( decoder . write ( Buffer . from ( chunk . buffer ) ) ) ; } else { bufferedChunks . push ( chunk ) ; bytesBuffered += chunk . byteLength ; if ( bytesBuffered >= minBytesRequiredForDetection ) { source . pause ( ) ; await createDecoder ( ) ; setTimeout ( ( ) => source . resume ( ) ) ; } } } ) ; source . on ( 'end' , async ( ) => { if ( ! decoder ) { await createDecoder ( ) ; } target . end ( decoder ?. end ( ) ) ; } ) ; } ) ; } encodeStream ( value , options ? ) ; encodeStream ( value ? , options ? ) ; async encodeStream ( value , options ? ) { let encoding = options ?. encoding ; const addBOM = options ?. hasBOM ; encoding = this . toIconvEncoding ( encoding ) ; if ( encoding === UTF8 && ! addBOM ) { return value === undefined ? undefined : typeof value === 'string' ? BinaryBuffer . fromString ( value ) : BinaryBufferReadable . fromReadable ( value ) ; } value = value || '' ; const readable = typeof value === 'string' ? Readable . fromString ( value ) : value ; const encoder = iconv . getEncoder ( encoding , { addBOM } ) ; let bytesWritten = false ; let done = false ; return { read ( ) { if ( done ) { return null ; } const chunk = readable . read ( ) ; if ( typeof chunk !== 'string' ) { done = true ; if ( ! bytesWritten && addBOM ) { switch ( encoding ) { case UTF8 : case UTF8_with_bom : return BinaryBuffer . wrap ( Uint8Array . from ( UTF8_BOM ) ) ; case UTF16be : return BinaryBuffer . wrap ( Uint8Array . from ( UTF16be_BOM ) ) ; case UTF16le : return BinaryBuffer . wrap ( Uint8Array . from ( UTF16le_BOM ) ) ; } } const leftovers = encoder . end ( ) ; if ( leftovers && leftovers . length > 0 ) { bytesWritten = true ; return BinaryBuffer . wrap ( leftovers ) ; } return null ; } bytesWritten = true ; return BinaryBuffer . wrap ( encoder . write ( chunk ) ) ; } } ; } }