import { AsyncLocalStorage } from "async_hooks" ; import type { OutgoingHttpHeaders } from "http" ; import * as querystring from "querystring" ; import * as git from "@fluidframework/gitresources" ; import { IGetRefParamsExternal , ICreateRefParamsExternal , IPatchRefParamsExternal , IWholeSummaryPayload , IWriteSummaryResponse , BasicRestWrapper , RestWrapper , IWholeFlatSummary , IWholeSummaryPayloadType , } from "@fluidframework/server-services-client" ; import { ITenantStorage , runWithRetry } from "@fluidframework/server-services-core" ; import * as uuid from "uuid" ; import * as winston from "winston" ; import { getCorrelationId } from "@fluidframework/server-services-utils" ; import { BaseTelemetryProperties , Lumberjack } from "@fluidframework/server-services-telemetry" ; import { getRequestErrorTranslator } from "../utils" ; import { ICache } from "./definitions" ; const packageDetails = require ( "../../package.json" ) ; const userAgent = `Historian/${ packageDetails . version }` ; export interface IDocument { existing : boolean ; docPrivateKey : string ; docPublicKey : string ; } function endsWith ( value , endings ) { for ( const ending of endings ) { if ( value . endsWith ( ending ) ) { return true ; } } return false ; } export class RestGitService { private readonly restWrapper : RestWrapper ; private readonly lumberProperties : Record < BaseTelemetryProperties , any > ; constructor ( private readonly storage , private readonly writeToExternalStorage , private readonly tenantId , private readonly documentId , private readonly cache ? , private readonly asyncLocalStorage ? ) { const defaultHeaders = { "User-Agent" : userAgent , "Storage-Routing-Id" : this . getStorageRoutingHeaderValue ( ) , } ; if ( storage . credentials ) { const token = Buffer . from ( `${ storage . credentials . user }:${ storage . credentials . password }` ) ; defaultHeaders . Authorization = `Basic ${ token . toString ( "base64" ) }` ; } this . lumberProperties = { [ BaseTelemetryProperties . tenantId ] : this . tenantId , [ BaseTelemetryProperties . documentId ] : this . documentId , } ; winston . info ( `Created RestGitService: ${ JSON . stringify ( { "BaseUrl" : storage . url , "Storage-Routing-Id" : this . getStorageRoutingHeaderValue ( ) , } ) }` ) ; Lumberjack . info ( `Created RestGitService: ${ JSON . stringify ( { "BaseUrl" : storage . url , "Storage-Routing-Id" : this . getStorageRoutingHeaderValue ( ) , } ) }` , this . lumberProperties ) ; this . restWrapper = new BasicRestWrapper ( storage . url , undefined , undefined , undefined , defaultHeaders , undefined , undefined , undefined , ( ) => getCorrelationId ( this . asyncLocalStorage ) || uuid . v4 ( ) ) ; } public async getBlob ( sha , useCache ) { return this . resolve ( sha , async ( ) => this . get < git . IBlob > ( `/repos/${ this . getRepoPath ( ) }/git/blobs/${ encodeURIComponent ( sha ) }` ) , useCache ) ; } public async createBlob ( blob ) { const createResults = await this . post < git . ICreateBlobResponse > ( `/repos/${ this . getRepoPath ( ) }/git/blobs` , blob ) ; this . getBlob ( createResults . sha , true ) . catch ( ( error ) => { winston . error ( `Error fetching blob ${ createResults . sha }` ) ; Lumberjack . error ( `Error fetching blob: ${ createResults . sha }` , this . lumberProperties ) ; } ) ; return createResults ; } public async getContent ( path , ref ) { const query = querystring . stringify ( { ref } ) ; return this . get ( `/repos/${ this . getRepoPath ( ) }/contents/${ encodeURIComponent ( path ) }?${ query }` ) ; } public async getCommits ( sha , count ) { let config ; if ( this . writeToExternalStorage ) { const getRefParams = { config : { enabled : true } , } ; config = encodeURIComponent ( JSON . stringify ( getRefParams ) ) ; } const query = querystring . stringify ( { count , sha , config , } ) ; return this . get ( `/repos/${ this . getRepoPath ( ) }/commits?${ query }` ) ; } public async getCommit ( sha , useCache ) { return this . resolve ( sha , async ( ) => this . get < git . ICommit > ( `/repos/${ this . getRepoPath ( ) }/git/commits/${ encodeURIComponent ( sha ) }` ) , useCache ) ; } public async createCommit ( commitParams ) { const commit = await this . post < git . ICommit > ( `/repos/${ this . getRepoPath ( ) }/git/commits` , commitParams ) ; this . setCache ( commit . sha , commit ) ; this . getTree ( commit . tree . sha , true , true ) . catch ( ( error ) => { winston . error ( `Error fetching commit tree ${ commit . tree . sha }` ) ; Lumberjack . error ( `Error fetching commit tree: ${ commit . tree . sha }` , this . lumberProperties ) ; } ) ; this . getHeader ( commit . sha , true ) . catch ( ( error ) => { winston . error ( `Error fetching header ${ commit . sha }` ) ; Lumberjack . error ( `Error fetching header: ${ commit . sha }` , this . lumberProperties ) ; } ) ; return commit ; } public async getRefs ( ) { return this . get ( `/repos/${ this . getRepoPath ( ) }/git/refs` ) ; } public async getRef ( ref ) { if ( this . writeToExternalStorage ) { const getRefParams = { config : { enabled : true } , } ; const params = encodeURIComponent ( JSON . stringify ( getRefParams ) ) ; return this . get ( `/repos/${ this . getRepoPath ( ) }/git/refs/${ encodeURIComponent ( ref ) }?config=${ params }` ) ; } return this . get ( `/repos/${ this . getRepoPath ( ) }/git/refs/${ encodeURIComponent ( ref ) }` ) ; } public async createRef ( params ) { if ( ! this . writeToExternalStorage ) { params . config . enabled = false ; } return this . post ( `/repos/${ this . getRepoPath ( ) }/git/refs` , params ) ; } public async createSummary ( summaryParams ) { const summaryResponse = await this . post < IWholeFlatSummary | IWriteSummaryResponse > ( `/repos/${ this . getRepoPath ( ) }/git/summaries` , summaryParams ) ; if ( summaryParams . type === "container" && ( summaryResponse as IWholeFlatSummary ) . trees !== undefined ) { this . setCache < IWholeFlatSummary > ( this . getSummaryCacheKey ( summaryParams . type ) , ( summaryResponse as IWholeFlatSummary ) ) ; } else { this . deleteFromCache ( this . getSummaryCacheKey ( summaryParams . type ) ) ; } return { id : summaryResponse . id } ; } public async deleteSummary ( softDelete ) { const headers = { "Soft-Delete" : softDelete } ; this . deleteFromCache ( this . getSummaryCacheKey ( "channel" ) ) ; this . deleteFromCache ( this . getSummaryCacheKey ( "container" ) ) ; return this . delete < boolean > ( `/repos/${ this . getRepoPath ( ) }/git/summaries` , headers ) ; } public async getSummary ( sha , useCache ) { return this . resolve ( this . getSummaryCacheKey ( "container" ) , async ( ) => this . get < IWholeFlatSummary > ( `/repos/${ this . getRepoPath ( ) }/git/summaries/${ encodeURIComponent ( sha ) }` ) , useCache ) ; } public async updateRef ( ref , params ) { if ( ! this . writeToExternalStorage ) { params . config . enabled = false ; } return this . patch ( `/repos/${ this . getRepoPath ( ) }/git/refs/${ ref }` , params ) ; } public async deleteRef ( ref ) { return this . delete ( `/repos/${ this . getRepoPath ( ) }/git/refs/${ ref }` ) ; } public async createTag ( tag ) { return this . post ( `/repos/${ this . getRepoPath ( ) }/git/tags` , tag ) ; } public async getTag ( tag ) { return this . get ( `/repos/${ this . getRepoPath ( ) }/git/tags/${ tag }` ) ; } public async createTree ( treeParams ) { const tree = await this . post < git . ITree > ( `/repos/${ this . getRepoPath ( ) }/git/trees` , treeParams ) ; this . setCache ( tree . sha , tree ) ; return tree ; } public async getTree ( sha , recursive , useCache ) { const key = recursive ? `${ sha }:recursive` : sha ; return this . resolve ( key , async ( ) => { const query = querystring . stringify ( { recursive : recursive ? 1 : 0 } ) ; return this . get < git . ITree > ( `/repos/${ this . getRepoPath ( ) }/git/trees/${ encodeURIComponent ( sha ) }?${ query }` ) ; } , useCache ) ; } public async getHeader ( sha , useCache ) { const version = await this . getCommit ( sha , useCache ) ; const key = `${ version . sha }:header` ; return this . resolve ( key , async ( ) => { const tree = await this . getTree ( version . tree . sha , true , useCache ) ; const blobs = await this . getHeaderBlobs ( tree , useCache ) ; return { blobs , tree , } ; } , useCache ) ; } public async getFullTree ( sha , useCache ) { const version = await this . getCommit ( sha , useCache ) ; const key = `${ version . sha }:tree` ; return this . resolve ( key , async ( ) => { const blobs = new Map < string , git . IBlob > ( ) ; const trees = new Map < string , git . ITree > ( ) ; const commits = new Map < string , git . ICommit > ( ) ; const baseTree = await this . getTree ( version . tree . sha , true , useCache ) ; commits . set ( version . sha , version ) ; trees . set ( baseTree . sha , baseTree ) ; const submoduleCommits = new Array < string > ( ) ; const quorumValuesSha = new Array < string > ( ) ; let quorumValues ; baseTree . tree . forEach ( ( entry ) => { if ( entry . path . includes ( "quorum" ) ) { quorumValuesSha . push ( entry . sha ) ; } if ( entry . path === "quorumValues" ) { quorumValues = entry . sha ; } if ( entry . type === "commit" ) { submoduleCommits . push ( entry . sha ) ; } } ) ; const submodulesP = Promise . all ( submoduleCommits . map ( async ( submoduleCommitSha ) => { const submoduleCommit = await this . getCommit ( submoduleCommitSha , useCache ) ; const submoduleTree = await this . getTree ( submoduleCommit . tree . sha , true , useCache ) ; trees . set ( submoduleCommit . tree . sha , submoduleTree ) ; commits . set ( submoduleCommit . sha , submoduleCommit ) ; } ) ) ; const blobsP = Promise . all ( quorumValuesSha . map ( async ( quorumSha ) => { const blob = await this . getBlob ( quorumSha , useCache ) ; blobs . set ( blob . sha , blob ) ; } ) ) ; await Promise . all ( [ submodulesP , blobsP ] ) ; return { blobs : Array . from ( blobs . values ( ) ) , commits : Array . from ( commits . values ( ) ) , quorumValues , trees : Array . from ( trees . values ( ) ) , } ; } , useCache ) ; } private getStorageRoutingHeaderValue ( ) { return `${ this . tenantId }:${ this . documentId }` ; } /**
     * Helper method to translate from an owner repo pair to the URL component for it. In the future we will require
     * the owner parameter. But for back compat we allow it to be optional.
     */ private getRepoPath ( ) { return `${ encodeURIComponent ( this . storage . owner ) }/${ encodeURIComponent ( this . storage . repository ) }` ; } private async getHeaderBlobs ( tree , useCache ) { const includeBlobs = [ ".attributes" , ".messages" , "header" ] ; const blobsP = [ ] ; for ( const entry of tree . tree ) { if ( entry . type === "blob" && endsWith ( entry . path , includeBlobs ) ) { const blobP = this . getBlob ( entry . sha , useCache ) ; blobsP . push ( blobP ) ; } } return Promise . all ( blobsP ) ; } private async get < T > ( url ) { return this . restWrapper . get < T > ( url ) . catch ( getRequestErrorTranslator ( url , "GET" , this . lumberProperties ) ) ; } private async post < T > ( url , requestBody ) { return this . restWrapper . post < T > ( url , requestBody , undefined , { "Content-Type" : "application/json" , } ) . catch ( getRequestErrorTranslator ( url , "POST" , this . lumberProperties ) ) ; } private async delete < T > ( url , headers ? ) { return this . restWrapper . delete < T > ( url , undefined , headers ) . catch ( getRequestErrorTranslator ( url , "DELETE" , this . lumberProperties ) ) ; } private async patch < T > ( url , requestBody ) { return this . restWrapper . patch < T > ( url , requestBody , undefined , { "Content-Type" : "application/json" , } ) . catch ( getRequestErrorTranslator ( url , "PATCH" , this . lumberProperties ) ) ; } /**
     * Caches the given key/value pair. Will log any errors with the cache.
     */ private setCache < T > ( key , value ) { if ( this . cache ) { runWithRetry ( async ( ) => this . cache . set ( key , value ) , "RestGitService.setCache" , 3 , 1000 , winston ) . catch ( ( error ) => { winston . error ( `Error caching ${ key } to redis` , error ) ; Lumberjack . error ( `Error caching ${ key } to redis` , this . lumberProperties , error ) ; } ) ; } } /**
     * Deletes the given key from the cache. Will log any errors with the cache.
     */ private deleteFromCache ( key ) { if ( this . cache ) { this . cache . delete ( key ) . catch ( ( error ) => { winston . error ( `Error deleting key ${ key } from Redis cache` , error ) ; Lumberjack . error ( `Error deleting key ${ key } from Redis cache` , this . lumberProperties , error ) ; } ) ; } } private async resolve < T > ( key , fetch , useCache ) { if ( this . cache && useCache ) { const cachedValue = await this . cache . get < T > ( key ) . catch ( ( error ) => { winston . error ( `Error fetching ${ key } from cache` , error ) ; Lumberjack . error ( `Error fetching ${ key } from cache` , this . lumberProperties , error ) ; return undefined ; } ) ; if ( cachedValue ) { winston . info ( `Resolving ${ key } from cache` ) ; Lumberjack . info ( `Resolving ${ key } from cache` , this . lumberProperties ) ; return cachedValue ; } winston . info ( `Fetching ${ key }` ) ; Lumberjack . info ( `Fetching ${ key }` , this . lumberProperties ) ; const value = await fetch ( ) ; this . setCache ( key , value ) ; return value ; } else { return fetch ( ) ; } } private getSummaryCacheKey ( type ) { return `${ this . tenantId }:${ this . documentId }:summary:${ type }` ; } }