import * as admin from "firebase-admin" ; import * as functions from "firebase-functions" ; import * as moment from "moment" ; import * as path from "path" ; import StreetPassRecord from "./types/StreetPassRecord" ; import config from "../config" ; import { decryptTempID } from "./getTempIDs" ; import { validateToken } from "./getUploadToken" ; import { getAllEncryptionKeys } from "./utils/getEncryptionKey" ; import formatTimestamp from "./utils/formatTimestamp" ; import { storeUploadLog } from "./utils/AuditLogger" ; /**
 * Process user's uploaded data.
 *
 * Most important tasks:
 *  + Validate upload token to get uid
 *  + Post-process records (e.g., validate exchanged messages, decrypt TempIDs)
 *  + Forward data for further processing
 */ export default async function processUploadedData ( object ) { const filePath = object . name ; console . log ( 'processUploadedData:' , 'Detected new file:' , filePath ) ; if ( filePath !== undefined && filePath . startsWith ( config . upload . recordsDir ) && filePath . endsWith ( '.json' ) ) { const fileName = path . basename ( filePath , '.json' ) ; let archiveFilePath = filePath ; console . log ( 'processUploadedData:' , 'File is streetPassRecords, content type:' , object . contentType ) ; await storeUploadLog ( fileName , { fileName : fileName , status : 'STARTED' , loggedTime : Date . now ( ) / 1000 } ) ; const step = "0 - move file" ; try { const uploadFile = admin . storage ( ) . bucket ( object . bucket ) . file ( filePath ) ; console . log ( 'processUploadedData:' , 'Uploaded file md5Hash' , object . md5Hash ) ; if ( ! archiveFilePath . startsWith ( `${ config . upload . recordsDir }/20` ) ) { archiveFilePath = archiveFilePath . replace ( config . upload . recordsDir , `${ config . upload . recordsDir }/${ formatTimestamp ( moment ( ) . unix ( ) , "YYYYMMDD" ) }` ) ; } const archivedFile = admin . storage ( ) . bucket ( config . upload . bucketForArchive ) . file ( archiveFilePath ) ; await uploadFile . copy ( archivedFile ) ; await uploadFile . delete ( ) ; console . log ( 'processUploadedData:' , `"step ${ step }"` , 'Uploaded file has been moved to archive folder.' ) ; } catch ( error ) { console . error ( new Error ( `processUploadedData: "step ${ step }" Error encountered, message: ${ error . message }. Stack trace:\n${ error . stack }` ) ) ; await storeUploadLog ( fileName , { fileName : fileName , id : '' , status : 'ERROR' , step : step , errorMessage : error . message , errorStackTrace : error . stack , loggedTime : Date . now ( ) / 1000 } ) ; return { status : 'ERROR' } ; } return _processUploadedData ( archiveFilePath ) ; } else { console . log ( 'processUploadedData:' , 'File is not streetPassRecords, ignore.' ) ; return { status : 'NONE' } ; } } export async function _processUploadedData ( filePath , validateTokenTimestamp = true ) { const fileName = path . basename ( filePath , '.json' ) ; let uid = '' , uploadCode = '' , step = '' ; try { step = "1 - load file" ; const { token , records , events } = JSON . parse ( await getStorageData ( config . upload . bucketForArchive , filePath ) ) ; console . log ( 'processUploadedData:' , `"step ${ step }"` , 'File is loaded, record count:' , records . length ) ; step = "2 - validate upload token" ; ( { uid , uploadCode } = await validateToken ( token , validateTokenTimestamp ) ) ; console . log ( 'processUploadedData:' , `"step ${ step }"` , 'Upload token is valid, id:' , uid ) ; step = "3 - post-process records" ; const validatedRecords = await validateRecords ( records ) ; console . log ( 'processUploadedData:' , `"step ${ step }"` , 'Complete validation of records,' , 'original count:' , records . length , 'after validation:' , validatedRecords . length ) ; step = "4 - forward data" ; await config . upload . dataForwarder . forwardData ( filePath , uid , uploadCode , validatedRecords , events ) ; await storeUploadLog ( fileName , { fileName : fileName , id : uid , status : 'SUCCESS' , uploadCode : uploadCode , recordsReceived : records . length , recordsSent : validatedRecords . length , loggedTime : Date . now ( ) / 1000 } ) ; } catch ( error ) { console . error ( new Error ( `processUploadedData: "step ${ step }" Error encountered, message: ${ error . message }. Stack trace:\n${ error . stack }` ) ) ; await storeUploadLog ( fileName , { fileName : fileName , id : uid , status : 'ERROR' , uploadCode : uploadCode , step : step , errorMessage : error . message , errorStackTrace : error . stack , loggedTime : Date . now ( ) / 1000 } ) ; return { status : 'ERROR' , message : error . message } ; } return { status : 'SUCCESS' , filePath : filePath } ; } param bucket param filePath async function getStorageData ( bucket , filePath ) { const archivedFile = admin . storage ( ) . bucket ( bucket ) . file ( filePath ) ; return archivedFile . download ( ) . then ( _ => _ . toString ( ) ) ; } /**
 * Validate records and convert temp ID to UID
 */ async function validateRecords ( records ) { if ( ! records ) { return [ ] ; } const encryptionKeys = await getAllEncryptionKeys ( ) ; records . forEach ( record => { record . timestamp = record . timestamp > 10000000000 ? record . timestamp / 1000 : record . timestamp ; record . timestampString = formatTimestamp ( record . timestamp ) ; validateRecord ( record , encryptionKeys ) ; } ) ; return records ; } param record param encryptionKeys function validateRecord ( record , encryptionKeys ) { record . isValid = false ; if ( ! record . msg ) { record . invalidReason = "no_msg" ; return ; } for ( const encryptionKey of encryptionKeys ) { try { const { uid , startTime , expiryTime } = decryptTempID ( record . msg , encryptionKey ) ; record . contactId = uid ; record . contactIdValidFrom = startTime ; record . contactIdValidTo = expiryTime ; if ( record . timestamp < startTime || record . timestamp > expiryTime ) { console . warn ( 'validateRecord:' , 'ID timestamp is not valid.' , 'ID startTime:' , formatTimestamp ( startTime ) , 'ID expiryTime:' , formatTimestamp ( expiryTime ) , 'timestamp:' , formatTimestamp ( record . timestamp ) ) ; record . isValid = false ; record . invalidReason = "expired_id" ; } else { record . isValid = true ; } break ; } catch ( error ) { console . warn ( 'validateRecord:' , 'Error while decrypting temp ID.' , error . message ) ; } } if ( ! record . isValid && ! record . invalidReason ) { record . contactId = record . msg ; record . invalidReason = "failed_decryption" ; } }