import { default as AbortController } from "abort-controller" ; import { v4 as uuid } from "uuid" ; import { ITelemetryLogger } from "@fluidframework/common-definitions" ; import { assert , fromUtf8ToBase64 , performance } from "@fluidframework/common-utils" ; import { DriverErrorType } from "@fluidframework/driver-definitions" ; import { PerformanceEvent } from "@fluidframework/telemetry-utils" ; import { IOdspResolvedUrl , ISnapshotOptions , OdspErrorType , InstrumentedStorageTokenFetcher , } from "@fluidframework/odsp-driver-definitions" ; import { ISnapshotTree } from "@fluidframework/protocol-definitions" ; import { IOdspSnapshot , ISnapshotCachedEntry , IVersionedValueWithEpoch , persistedCacheValueVersion } from "./contracts" ; import { getQueryString } from "./getQueryString" ; import { getUrlAndHeadersWithAuth } from "./getUrlAndHeadersWithAuth" ; import { fetchAndParseAsJSONHelper , fetchArray , getWithRetryForTokenRefresh , getWithRetryForTokenRefreshRepeat , IOdspResponse , ISnapshotContents , } from "./odspUtils" ; import { convertOdspSnapshotToSnapsohtTreeAndBlobs } from "./odspSnapshotParser" ; import { parseCompactSnapshotResponse } from "./compactSnapshotParser" ; import { ReadBuffer } from "./ReadBufferUtils" ; import { EpochTracker } from "./epochTracker" ; param snapshotUrl param token param storageFetchWrapper param versionId param fetchFullSnapshot returns export async function fetchSnapshot ( snapshotUrl , token , versionId , fetchFullSnapshot , logger , snapshotDownloader ) { const path = `/trees/${ versionId }` ; let queryParams = { } ; if ( fetchFullSnapshot ) { if ( versionId !== "latest" ) { queryParams = { channels : 1 , blobs : 2 } ; } else { queryParams = { deltas : 1 , channels : 1 , blobs : 2 } ; } } const queryString = getQueryString ( queryParams ) ; const { url , headers } = getUrlAndHeadersWithAuth ( `${ snapshotUrl }${ path }${ queryString }` , token ) ; const response = await PerformanceEvent . timedExecAsync ( logger , { eventName : "fetchSnapshot" , headers : Object . keys ( headers ) . length !== 0 ? true : undefined } , async ( ) => snapshotDownloader ( url , { headers } ) ) as IOdspResponse < IOdspSnapshot > ; return convertOdspSnapshotToSnapsohtTreeAndBlobs ( response . content ) ; } export async function fetchSnapshotWithRedeem ( odspResolvedUrl , storageTokenFetcher , snapshotOptions , logger , snapshotDownloader , putInCache , removeEntries , enableRedeemFallback ? ) { return fetchLatestSnapshotCore ( odspResolvedUrl , storageTokenFetcher , snapshotOptions , logger , snapshotDownloader , putInCache , enableRedeemFallback ) . catch ( async ( error ) => { if ( enableRedeemFallback && isRedeemSharingLinkError ( odspResolvedUrl , error ) ) { logger . sendErrorEvent ( { eventName : "RedeemFallback" , errorType : error . errorType } , error ) ; await redeemSharingLink ( odspResolvedUrl , storageTokenFetcher , logger ) ; const odspResolvedUrlWithoutShareLink = { ... odspResolvedUrl , sharingLinkToRedeem : undefined } ; if ( odspResolvedUrlWithoutShareLink . shareLinkInfo ) { odspResolvedUrlWithoutShareLink . shareLinkInfo = { ... odspResolvedUrlWithoutShareLink . shareLinkInfo , sharingLinkToRedeem : undefined } ; } return fetchLatestSnapshotCore ( odspResolvedUrlWithoutShareLink , storageTokenFetcher , snapshotOptions , logger , snapshotDownloader , putInCache ) ; } else { throw error ; } } ) . catch ( async ( error ) => { if ( typeof error === "object" && error !== null && error . errorType === DriverErrorType . authorizationError || error . errorType === DriverErrorType . fileNotFoundOrAccessDeniedError ) { await removeEntries ( ) ; } throw error ; } ) ; } async function redeemSharingLink ( odspResolvedUrl , storageTokenFetcher , logger ) { return PerformanceEvent . timedExecAsync ( logger , { eventName : "RedeemShareLink" } , async ( ) => getWithRetryForTokenRefresh ( async ( tokenFetchOptions ) => { assert ( ! ! odspResolvedUrl . sharingLinkToRedeem , 0x1ed ) ; const storageToken = await storageTokenFetcher ( tokenFetchOptions , "RedeemShareLink" ) ; const encodedShareUrl = getEncodedShareUrl ( odspResolvedUrl . sharingLinkToRedeem ) ; const redeemUrl = `${ odspResolvedUrl . siteUrl }/_api/v2.0/shares/${ encodedShareUrl }` ; const { url , headers } = getUrlAndHeadersWithAuth ( redeemUrl , storageToken ) ; headers . prefer = "redeemSharingLink" ; return fetchAndParseAsJSONHelper ( url , { headers } ) ; } ) ) ; } async function fetchLatestSnapshotCore ( odspResolvedUrl , storageTokenFetcher , snapshotOptions , logger , snapshotDownloader , putInCache , enableRedeemFallback ? ) { return getWithRetryForTokenRefresh ( async ( tokenFetchOptions ) => { const storageToken = await storageTokenFetcher ( tokenFetchOptions , "TreesLatest" , true ) ; assert ( storageToken !== null , 0x1e5 ) ; let controller ; if ( snapshotOptions ?. timeout !== undefined ) { controller = new AbortController ( ) ; setTimeout ( ( ) => controller ! . abort ( ) , snapshotOptions . timeout ) ; } const perfEvent = { eventName : "TreesLatest" , attempts : tokenFetchOptions . refresh ? 2 : 1 , shareLinkPresent : odspResolvedUrl . sharingLinkToRedeem !== undefined , redeemFallbackEnabled : enableRedeemFallback } ; if ( snapshotOptions !== undefined ) { Object . entries ( snapshotOptions ) . forEach ( ( [ key , value ] ) => { if ( value !== undefined ) { perfEvent [ `snapshotOption_${ key }` ] = value ; } } ) ; } return PerformanceEvent . timedExecAsync ( logger , perfEvent , async ( event ) => { const startTime = performance . now ( ) ; const response = await snapshotDownloader ( odspResolvedUrl , storageToken , snapshotOptions , controller ) ; const endTime = performance . now ( ) ; const overallTime = endTime - startTime ; const snapshot = response . odspSnapshotResponse . content ; let dnstime ; let redirectTime ; let tcpHandshakeTime ; let secureConntime ; let responseTime ; let fetchStToRespEndTime ; let reqStToRespEndTime ; let networkTime ; const spReqDuration = response . odspSnapshotResponse . headers . get ( "sprequestduration" ) ; const resources1 = performance . getEntriesByType ?. ( "resource" ) ?? [ ] ; for ( let i = resources1 . length - 1 ; i > 0 ; i -- ) { const indResTime = resources1 [ i ] as PerformanceResourceTiming ; const resource_name = indResTime . name ; const resource_initiatortype = indResTime . initiatorType ; if ( ( resource_initiatortype . localeCompare ( "fetch" ) === 0 ) && ( resource_name . localeCompare ( response . requestUrl ) === 0 ) ) { redirectTime = indResTime . redirectEnd - indResTime . redirectStart ; dnstime = indResTime . domainLookupEnd - indResTime . domainLookupStart ; tcpHandshakeTime = indResTime . connectEnd - indResTime . connectStart ; secureConntime = ( indResTime . secureConnectionStart > 0 ) ? ( indResTime . connectEnd - indResTime . secureConnectionStart ) : 0 ; responseTime = indResTime . responseEnd - indResTime . responseStart ; fetchStToRespEndTime = ( indResTime . fetchStart > 0 ) ? ( indResTime . responseEnd - indResTime . fetchStart ) : 0 ; reqStToRespEndTime = ( indResTime . requestStart > 0 ) ? ( indResTime . responseEnd - indResTime . requestStart ) : 0 ; networkTime = ( indResTime . startTime > 0 ) ? ( indResTime . responseEnd - indResTime . startTime ) : 0 ; if ( spReqDuration ) { networkTime = networkTime - parseInt ( spReqDuration , 10 ) ; } break ; } } const { numTrees , numBlobs , encodedBlobsSize } = validateAndEvalBlobsAndTrees ( response . odspSnapshotResponse . content ) ; const clientTime = networkTime ? overallTime - networkTime : undefined ; const canCache = response . odspSnapshotResponse . headers . get ( "disablebrowsercachingofusercontent" ) !== "true" ; const sequenceNumber = snapshot . sequenceNumber ?? 0 ; const seqNumberFromOps = snapshot . ops && snapshot . ops . length > 0 ? snapshot . ops [ 0 ] . sequenceNumber - 1 : undefined ; if ( ! Number . isInteger ( sequenceNumber ) || seqNumberFromOps !== undefined && seqNumberFromOps !== sequenceNumber ) { logger . sendErrorEvent ( { eventName : "fetchSnapshotError" , sequenceNumber , seqNumberFromOps } ) ; snapshot . sequenceNumber = undefined ; } else if ( canCache ) { const fluidEpoch = response . odspSnapshotResponse . headers . get ( "x-fluid-epoch" ) ; assert ( fluidEpoch !== undefined , 0x1e6 ) ; const value = { ... snapshot , cacheEntryTime : Date . now ( ) } ; const valueWithEpoch = { value , fluidEpoch , version : persistedCacheValueVersion } ; putInCache ( valueWithEpoch ) ; } event . end ( { trees : numTrees , blobs : snapshot . blobs ?. size ?? 0 , leafNodes : numBlobs , encodedBlobsSize , sequenceNumber , ops : snapshot . ops ?. length ?? 0 , headers : Object . keys ( response . requestHeaders ) . length !== 0 ? true : undefined , redirecttime : redirectTime , dnsLookuptime : dnstime , responsenetworkTime : responseTime , tcphandshakeTime : tcpHandshakeTime , secureconnectiontime : secureConntime , fetchstarttorespendtime : fetchStToRespEndTime , reqstarttorespendtime : reqStToRespEndTime , overalltime : overallTime , networktime : networkTime , clienttime : clientTime , sltelemetry : response . odspSnapshotResponse . headers . get ( "x-fluid-sltelemetry" ) , ... response . odspSnapshotResponse . commonSpoHeaders } ) ; return snapshot ; } ) . catch ( ( error ) => { if ( typeof error === "object" && error !== null && ( error . errorType === DriverErrorType . fetchFailure || error . errorType === OdspErrorType . fetchTimeout ) ) { error [ getWithRetryForTokenRefreshRepeat ] = true ; } throw error ; } ) ; } ) ; } interface ISnapshotRequestAndResponseOptions { odspSnapshotResponse : IOdspResponse < ISnapshotContents > ; requestUrl : string ; requestHeaders : { [ index ] : any ; } ; } param odspResolvedUrl param storageToken param snapshotOptions param controller param epochTracker returns async function fetchSnapshotContentsCoreV1 ( odspResolvedUrl , storageToken , snapshotOptions , controller ? , epochTracker ? ) { const snapshotUrl = odspResolvedUrl . endpoints . snapshotStorageUrl ; const url = `${ snapshotUrl }/trees/latest?ump=1` ; const { body , headers } = getFormBodyAndHeaders ( odspResolvedUrl , storageToken , snapshotOptions ) ; const fetchOptions = { body , headers , signal : controller ?. signal , method : "POST" } ; const response = await ( epochTracker ?. fetchAndParseAsJSON < IOdspSnapshot > ( url , fetchOptions , "treesLatest" , true ) ?? fetchAndParseAsJSONHelper < IOdspSnapshot > ( url , fetchOptions ) ) ; const snapshotContents = convertOdspSnapshotToSnapsohtTreeAndBlobs ( response . content ) ; const finalSnapshotContents = { ... response , content : snapshotContents } ; return { odspSnapshotResponse : finalSnapshotContents , requestHeaders : headers , requestUrl : url } ; } param odspResolvedUrl param storageToken param snapshotOptions param controller param epochTracker returns async function fetchSnapshotContentsCoreV2 ( odspResolvedUrl , storageToken , snapshotOptions , controller ? , epochTracker ? ) { const fullUrl = `${ odspResolvedUrl . siteUrl }/_api/v2.1/drives/${ odspResolvedUrl . driveId }/items/${ odspResolvedUrl . itemId }/opStream/attachments/latest/content?ump=1` ; const { body , headers } = getFormBodyAndHeaders ( odspResolvedUrl , storageToken , snapshotOptions ) ; const fetchOptions = { body , headers , signal : controller ?. signal , method : "POST" } ; const response = await ( epochTracker ?. fetchArray ( fullUrl , fetchOptions , "treesLatest" , true ) ?? fetchArray ( fullUrl , fetchOptions ) ) ; const snapshotContents = parseCompactSnapshotResponse ( new ReadBuffer ( new Uint8Array ( response . content ) ) ) ; const finalSnapshotContents = { ... response , content : snapshotContents } ; return { odspSnapshotResponse : finalSnapshotContents , requestHeaders : headers , requestUrl : fullUrl } ; } function getFormBodyAndHeaders ( odspResolvedUrl , storageToken , snapshotOptions ) { const formBoundary = uuid ( ) ; const formParams = [ ] ; formParams . push ( `--${ formBoundary }` ) ; formParams . push ( `Authorization: Bearer ${ storageToken }` ) ; formParams . push ( `X-HTTP-Method-Override: GET` ) ; if ( snapshotOptions !== undefined ) { Object . entries ( snapshotOptions ) . forEach ( ( [ key , value ] ) => { if ( value !== undefined ) { formParams . push ( `${ key }: ${ value }` ) ; } } ) ; } if ( odspResolvedUrl . sharingLinkToRedeem ) { formParams . push ( `sl: ${ odspResolvedUrl . sharingLinkToRedeem }` ) ; } formParams . push ( `_post: 1` ) ; formParams . push ( `\r\n--${ formBoundary }--` ) ; const postBody = formParams . join ( "\r\n" ) ; const headers = { "Content-Type" : `multipart/form-data;boundary=${ formBoundary }` } ; return { body : postBody , headers } ; } function validateAndEvalBlobsAndTrees ( snapshot ) { assert ( snapshot . snapshotTree !== undefined , 0x200 ) ; assert ( snapshot . blobs !== undefined , 0x201 ) ; const numTrees = countTreesInSnapshotTree ( snapshot . snapshotTree ) ; const numBlobs = snapshot . blobs . size ; let encodedBlobsSize = 0 ; for ( const [ _ , blobContent ] of snapshot . blobs ) { encodedBlobsSize += blobContent . byteLength ; } return { numTrees , numBlobs , encodedBlobsSize } ; } function countTreesInSnapshotTree ( snapshotTree ) { let numTrees = 0 ; for ( const [ _ , tree ] of Object . entries ( snapshotTree . trees ) ) { numTrees += 1 ; numTrees += countTreesInSnapshotTree ( tree ) ; } return numTrees ; } export async function downloadSnapshot ( odspResolvedUrl , storageToken , logger , snapshotOptions , fetchBinarySnapshotFormat ? , controller ? , epochTracker ? ) { if ( fetchBinarySnapshotFormat ) { logger . sendTelemetryEvent ( { eventName : "BinarySnapshotFetched" } ) ; return fetchSnapshotContentsCoreV2 ( odspResolvedUrl , storageToken , snapshotOptions , controller , epochTracker ) ; } else { return fetchSnapshotContentsCoreV1 ( odspResolvedUrl , storageToken , snapshotOptions , controller , epochTracker ) ; } } function isRedeemSharingLinkError ( odspResolvedUrl , error ) { if ( odspResolvedUrl . sharingLinkToRedeem !== undefined && ( typeof error === "object" && error !== null ) && ( error . errorType === DriverErrorType . authorizationError || error . errorType === DriverErrorType . fileNotFoundOrAccessDeniedError ) ) { return true ; } return false ; } function getEncodedShareUrl ( url ) { /**
     * Encode the url to accepted format by Sharepoint
     * https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/shares_get
     */ let encodedUrl = fromUtf8ToBase64 ( encodeURI ( url ) ) ; encodedUrl = encodedUrl . replace ( /=+$/g , "" ) . replace ( /\//g , "_" ) . replace ( /\+/g , "-" ) ; encodedUrl = "u!" . concat ( encodedUrl ) ; return encodedUrl ; }