import fs from "fs" ; import path from "path" ; import { zip } from "ramda" ; import { Readable } from "stream" ; import { Storage as GoogleCloudStorage , File , CreateReadStreamOptions , } from "@google-cloud/storage" ; import { AbstractAdapter } from "./AbstractAdapter" ; import { StorageType , ConfigGoogleCloud } from "./types" ; import { parseUrl } from "./util" ; export class AdapterGoogleCloudStorage extends AbstractAdapter { protected type = StorageType . GCS ; private bucketNames : string [ ] = [ ] ; private storage : GoogleCloudStorage ; public static defaultOptions = { slug : true , } ; constructor ( config ) { super ( ) ; const cfg = this . parseConfig ( config ) ; this . config = { ... cfg } ; if ( cfg . slug ) { this . slug = cfg . slug ; delete cfg . slug ; } if ( cfg . bucketName ) { this . bucketName = this . generateSlug ( cfg . bucketName , this . slug ) ; delete cfg . bucketName ; } delete cfg . type ; this . storage = new GoogleCloudStorage ( cfg ) ; } private parseConfig ( config ) { let cfg ; if ( typeof config === "string" ) { const { type , part1 : keyFilename , part2 : projectId , bucketName , queryString } = parseUrl ( config ) ; cfg = { type , keyFilename , projectId , bucketName , ... queryString , } ; } else { cfg = { ... config } ; } return cfg ; } async init ( ) { if ( this . bucketName ) { await this . createBucket ( this . bucketName ) ; this . bucketNames . push ( this . bucketName ) ; } this . initialized = true ; return Promise . resolve ( true ) ; } async getFile ( fileName , retries = 5 ) { const file = this . storage . bucket ( this . bucketName ) . file ( fileName ) ; const [ exists ] = await file . exists ( ) ; if ( ! exists && retries !== 0 ) { const r = retries - 1 ; await new Promise ( res => { setTimeout ( res , 250 ) ; } ) ; return this . getFile ( fileName , r ) ; } if ( ! exists ) { throw new Error ( `File ${ fileName } could not be retrieved from bucket ${ this . bucketName }` ) ; } return file ; } async getFileAsReadable ( fileName , options = { start : 0 } ) { const file = this . storage . bucket ( this . bucketName ) . file ( fileName ) ; const [ exists ] = await file . exists ( ) ; if ( exists ) { return file . createReadStream ( options ) ; } throw new Error ( `File ${ fileName } could not be retrieved from bucket ${ this . bucketName }` ) ; } async downloadFile ( fileName , downloadPath ) { const file = this . storage . bucket ( this . bucketName ) . file ( fileName ) ; const localFilename = path . join ( downloadPath , fileName ) ; await file . download ( { destination : localFilename } ) ; } async removeFile ( fileName ) { try { await this . storage . bucket ( this . bucketName ) . file ( fileName ) . delete ( ) ; return "file deleted" ; } catch ( e ) { if ( e . message . indexOf ( "No such object" ) !== - 1 ) { return "file deleted" ; } throw e ; } } protected async store ( buffer , targetPath ) ; protected async store ( stream , targetPath ) ; protected async store ( origPath , targetPath ) ; protected async store ( arg , targetPath ) { if ( ! this . bucketName ) { throw new Error ( "no bucket selected" ) ; } await this . createBucket ( this . bucketName ) ; let readStream ; if ( typeof arg === "string" ) { await fs . promises . stat ( arg ) ; readStream = fs . createReadStream ( arg ) ; } else if ( arg instanceof Buffer ) { readStream = new Readable ( ) ; readStream . _read = ( ) : void => { } ; readStream . push ( arg ) ; readStream . push ( null ) ; } else if ( arg instanceof Readable ) { readStream = arg ; } const writeStream = this . storage . bucket ( this . bucketName ) . file ( targetPath ) . createWriteStream ( ) ; return new Promise ( ( resolve , reject ) => { readStream . pipe ( writeStream ) . on ( "error" , reject ) . on ( "finish" , resolve ) ; writeStream . on ( "error" , reject ) ; } ) ; } async createBucket ( name ) { const msg = this . validateName ( name ) ; if ( msg !== null ) { return Promise . reject ( msg ) ; } const n = this . generateSlug ( name ) ; if ( this . bucketNames . findIndex ( b => b === n ) !== - 1 ) { return "bucket exists" ; } try { const bucket = this . storage . bucket ( n ) ; const [ exists ] = await bucket . exists ( ) ; if ( exists ) { return "bucket exists" ; } } catch ( e ) { } try { await this . storage . createBucket ( n ) ; this . bucketNames . push ( n ) ; return "bucket created" ; } catch ( e ) { if ( e . code === 409 && e . message === "You already own this bucket. Please select another name." ) { return ; } throw new Error ( e . message ) ; } } async selectBucket ( name ) { if ( name === null ) { this . bucketName = "" ; return "bucket deselected" ; } this . createBucket ( name ) . then ( ( ) => { this . bucketName = name ; return "bucket selected" ; } ) . catch ( e => { throw e ; } ) ; } async clearBucket ( name ? ) { let n = name || this . bucketName ; n = this . generateSlug ( n ) ; await this . storage . bucket ( n ) . deleteFiles ( { force : true } ) ; return "bucket cleared" ; } async deleteBucket ( name ? ) { let n = name || this . bucketName ; n = this . generateSlug ( n ) ; await this . clearBucket ( n ) ; const data = await this . storage . bucket ( n ) . delete ( ) ; if ( n === this . bucketName ) { this . bucketName = "" ; } this . bucketNames = this . bucketNames . filter ( b => b !== n ) ; return "bucket deleted" ; } async listBuckets ( ) { const [ buckets ] = await this . storage . getBuckets ( ) ; this . bucketNames = buckets . map ( b => b . metadata . id ) ; return this . bucketNames ; } private async getMetaData ( files ) { const sizes = [ ] ; for ( let i = 0 ; i < files . length ; i += 1 ) { const file = this . storage . bucket ( this . bucketName ) . file ( files [ i ] ) ; const [ metadata ] = await file . getMetadata ( ) ; sizes . push ( parseInt ( metadata . size , 10 ) ) ; } return sizes ; } async listFiles ( numFiles = 1000 ) { if ( ! this . bucketName ) { throw new Error ( "no bucket selected" ) ; } const data = await this . storage . bucket ( this . bucketName ) . getFiles ( ) ; const names = data [ 0 ] . map ( f => f . name ) ; const sizes = await this . getMetaData ( names ) ; return zip ( names , sizes ) as [ string , number ] [ ] ; } async sizeOf ( name ) { if ( ! this . bucketName ) { throw new Error ( "no bucket selected" ) ; } const file = this . storage . bucket ( this . bucketName ) . file ( name ) ; const [ metadata ] = await file . getMetadata ( ) ; return parseInt ( metadata . size , 10 ) ; } async fileExists ( name ) { const data = await this . storage . bucket ( this . bucketName ) . file ( name ) . exists ( ) ; return data [ 0 ] ; } }