import express from 'express' ; import { move , readFile } from 'fs-extra' ; import { decode } from 'magnet-uri' ; import parseTorrent , { Instance } from 'parse-torrent' ; import { join } from 'path' ; import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' ; import { Hooks } from '@server/lib/plugins/hooks' ; import { ServerConfigManager } from '@server/lib/server-config-manager' ; import { setVideoTags } from '@server/lib/video' ; import { FilteredModelAttributes } from '@server/types' ; import { MChannelAccountDefault , MThumbnail , MUser , MVideoAccountDefault , MVideoCaption , MVideoTag , MVideoThumbnail , MVideoWithBlacklistLight } from '@server/types/models' ; import { MVideoImportFormattable } from '@server/types/models/video/video-import' ; import { ServerErrorCode , ThumbnailType , VideoImportCreate , VideoImportState , VideoPrivacy , VideoState } from '@shared/models' ; import { auditLoggerFactory , getAuditIdFromRes , VideoImportAuditView } from '../../../helpers/audit-logger' ; import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' ; import { isArray } from '../../../helpers/custom-validators/misc' ; import { cleanUpReqFiles , createReqFiles } from '../../../helpers/express-utils' ; import { logger } from '../../../helpers/logger' ; import { getSecureTorrentName } from '../../../helpers/utils' ; import { YoutubeDLInfo , YoutubeDLWrapper } from '../../../helpers/youtube-dl' ; import { CONFIG } from '../../../initializers/config' ; import { MIMETYPES } from '../../../initializers/constants' ; import { sequelizeTypescript } from '../../../initializers/database' ; import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url' ; import { JobQueue } from '../../../lib/job-queue/job-queue' ; import { updateVideoMiniatureFromExisting , updateVideoMiniatureFromUrl } from '../../../lib/thumbnail' ; import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' ; import { asyncMiddleware , asyncRetryTransactionMiddleware , authenticate , videoImportAddValidator } from '../../../middlewares' ; import { VideoModel } from '../../../models/video/video' ; import { VideoCaptionModel } from '../../../models/video/video-caption' ; import { VideoImportModel } from '../../../models/video/video-import' ; const auditLogger = auditLoggerFactory ( 'video-imports' ) ; const videoImportsRouter = express . Router ( ) ; const reqVideoFileImport = createReqFiles ( [ 'thumbnailfile' , 'previewfile' , 'torrentfile' ] , Object . assign ( { } , MIMETYPES . TORRENT . MIMETYPE_EXT , MIMETYPES . IMAGE . MIMETYPE_EXT ) , { thumbnailfile : CONFIG . STORAGE . TMP_DIR , previewfile : CONFIG . STORAGE . TMP_DIR , torrentfile : CONFIG . STORAGE . TMP_DIR } ) ; videoImportsRouter . post ( '/imports' , authenticate , reqVideoFileImport , asyncMiddleware ( videoImportAddValidator ) , asyncRetryTransactionMiddleware ( addVideoImport ) ) ; export { videoImportsRouter } ; function addVideoImport ( req , res ) { if ( req . body . targetUrl ) return addYoutubeDLImport ( req , res ) ; const file = req . files ?. [ 'torrentfile' ] ?. [ 0 ] ; if ( req . body . magnetUri || file ) return addTorrentImport ( req , res , file ) ; } async function addTorrentImport ( req , res , torrentfile ) { const body = req . body ; const user = res . locals . oauth . token . User ; let videoName ; let torrentName ; let magnetUri ; if ( torrentfile ) { const result = await processTorrentOrAbortRequest ( req , res , torrentfile ) ; if ( ! result ) return ; videoName = result . name ; torrentName = result . torrentName ; } else { const result = processMagnetURI ( body ) ; magnetUri = result . magnetUri ; videoName = result . name ; } const video = await buildVideo ( res . locals . videoChannel . id , body , { name : videoName } ) ; const thumbnailModel = await processThumbnail ( req , video ) ; const previewModel = await processPreview ( req , video ) ; const videoImport = await insertIntoDB ( { video , thumbnailModel , previewModel , videoChannel : res . locals . videoChannel , tags : body . tags || undefined , user , videoImportAttributes : { magnetUri , torrentName , state : VideoImportState . PENDING , userId : user . id } } ) ; const payload = { type : torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri' , videoImportId : videoImport . id , magnetUri } ; await JobQueue . Instance . createJobWithPromise ( { type : 'video-import' , payload } ) ; auditLogger . create ( getAuditIdFromRes ( res ) , new VideoImportAuditView ( videoImport . toFormattedJSON ( ) ) ) ; return res . json ( videoImport . toFormattedJSON ( ) ) . end ( ) ; } async function addYoutubeDLImport ( req , res ) { const body = req . body ; const targetUrl = body . targetUrl ; const user = res . locals . oauth . token . User ; const youtubeDL = new YoutubeDLWrapper ( targetUrl , ServerConfigManager . Instance . getEnabledResolutions ( 'vod' ) ) ; let youtubeDLInfo ; try { youtubeDLInfo = await youtubeDL . getInfoForDownload ( ) ; } catch ( err ) { logger . info ( 'Cannot fetch information from import for URL %s.' , targetUrl , { err } ) ; return res . fail ( { message : 'Cannot fetch remote information of this URL.' , data : { targetUrl } } ) ; } const video = await buildVideo ( res . locals . videoChannel . id , body , youtubeDLInfo ) ; let thumbnailModel = await processThumbnail ( req , video ) ; if ( ! thumbnailModel && youtubeDLInfo . thumbnailUrl ) { try { thumbnailModel = await processThumbnailFromUrl ( youtubeDLInfo . thumbnailUrl , video ) ; } catch ( err ) { logger . warn ( 'Cannot process thumbnail %s from youtubedl.' , youtubeDLInfo . thumbnailUrl , { err } ) ; } } let previewModel = await processPreview ( req , video ) ; if ( ! previewModel && youtubeDLInfo . thumbnailUrl ) { try { previewModel = await processPreviewFromUrl ( youtubeDLInfo . thumbnailUrl , video ) ; } catch ( err ) { logger . warn ( 'Cannot process preview %s from youtubedl.' , youtubeDLInfo . thumbnailUrl , { err } ) ; } } const videoImport = await insertIntoDB ( { video , thumbnailModel , previewModel , videoChannel : res . locals . videoChannel , tags : body . tags || youtubeDLInfo . tags , user , videoImportAttributes : { targetUrl , state : VideoImportState . PENDING , userId : user . id } } ) ; await processYoutubeSubtitles ( youtubeDL , targetUrl , video . id ) ; let fileExt = `.${ youtubeDLInfo . ext }` ; if ( ! isVideoFileExtnameValid ( fileExt ) ) fileExt = '.mp4' ; const payload = { type : 'youtube-dl' as 'youtube-dl' , videoImportId : videoImport . id , fileExt } ; await JobQueue . Instance . createJobWithPromise ( { type : 'video-import' , payload } ) ; auditLogger . create ( getAuditIdFromRes ( res ) , new VideoImportAuditView ( videoImport . toFormattedJSON ( ) ) ) ; return res . json ( videoImport . toFormattedJSON ( ) ) . end ( ) ; } async function buildVideo ( channelId , body , importData ) { let videoData = { name : body . name || importData . name || 'Unknown name' , remote : false , category : body . category || importData . category , licence : body . licence ?? importData . licence ?? CONFIG . DEFAULTS . PUBLISH . LICENCE , language : body . language || importData . language , commentsEnabled : body . commentsEnabled ?? CONFIG . DEFAULTS . PUBLISH . COMMENTS_ENABLED , downloadEnabled : body . downloadEnabled ?? CONFIG . DEFAULTS . PUBLISH . DOWNLOAD_ENABLED , waitTranscoding : body . waitTranscoding || false , state : VideoState . TO_IMPORT , nsfw : body . nsfw || importData . nsfw || false , description : body . description || importData . description , support : body . support || null , privacy : body . privacy || VideoPrivacy . PRIVATE , duration : 0 , channelId : channelId , originallyPublishedAt : body . originallyPublishedAt ? new Date ( body . originallyPublishedAt ) : importData . originallyPublishedAt } ; videoData = await Hooks . wrapObject ( videoData , body . targetUrl ? 'filter:api.video.import-url.video-attribute.result' : 'filter:api.video.import-torrent.video-attribute.result' ) ; const video = new VideoModel ( videoData ) ; video . url = getLocalVideoActivityPubUrl ( video ) ; return video ; } async function processThumbnail ( req , video ) { const thumbnailField = req . files ? req . files [ 'thumbnailfile' ] : undefined ; if ( thumbnailField ) { const thumbnailPhysicalFile = thumbnailField [ 0 ] ; return updateVideoMiniatureFromExisting ( { inputPath : thumbnailPhysicalFile . path , video , type : ThumbnailType . MINIATURE , automaticallyGenerated : false } ) ; } return undefined ; } async function processPreview ( req , video ) { const previewField = req . files ? req . files [ 'previewfile' ] : undefined ; if ( previewField ) { const previewPhysicalFile = previewField [ 0 ] ; return updateVideoMiniatureFromExisting ( { inputPath : previewPhysicalFile . path , video , type : ThumbnailType . PREVIEW , automaticallyGenerated : false } ) ; } return undefined ; } async function processThumbnailFromUrl ( url , video ) { try { return updateVideoMiniatureFromUrl ( { downloadUrl : url , video , type : ThumbnailType . MINIATURE } ) ; } catch ( err ) { logger . warn ( 'Cannot generate video thumbnail %s for %s.' , url , video . url , { err } ) ; return undefined ; } } async function processPreviewFromUrl ( url , video ) { try { return updateVideoMiniatureFromUrl ( { downloadUrl : url , video , type : ThumbnailType . PREVIEW } ) ; } catch ( err ) { logger . warn ( 'Cannot generate video preview %s for %s.' , url , video . url , { err } ) ; return undefined ; } } async function insertIntoDB ( parameters ) { const { video , thumbnailModel , previewModel , videoChannel , tags , videoImportAttributes , user } = parameters ; const videoImport = await sequelizeTypescript . transaction ( async ( t ) => { const sequelizeOptions = { transaction : t } ; const videoCreated = await video . save ( sequelizeOptions ) as ( MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag ) ; videoCreated . VideoChannel = videoChannel ; if ( thumbnailModel ) await videoCreated . addAndSaveThumbnail ( thumbnailModel , t ) ; if ( previewModel ) await videoCreated . addAndSaveThumbnail ( previewModel , t ) ; await autoBlacklistVideoIfNeeded ( { video : videoCreated , user , notify : false , isRemote : false , isNew : true , transaction : t } ) ; await setVideoTags ( { video : videoCreated , tags , transaction : t } ) ; const videoImport = await VideoImportModel . create ( Object . assign ( { videoId : videoCreated . id } , videoImportAttributes ) , sequelizeOptions ) as MVideoImportFormattable ; videoImport . Video = videoCreated ; return videoImport ; } ) ; return videoImport ; } async function processTorrentOrAbortRequest ( req , res , torrentfile ) { const torrentName = torrentfile . originalname ; const newTorrentPath = join ( CONFIG . STORAGE . TORRENTS_DIR , getSecureTorrentName ( torrentName ) ) ; await move ( torrentfile . path , newTorrentPath , { overwrite : true } ) ; torrentfile . path = newTorrentPath ; const buf = await readFile ( torrentfile . path ) ; const parsedTorrent = parseTorrent ( buf ) as Instance ; if ( parsedTorrent . files . length !== 1 ) { cleanUpReqFiles ( req ) ; res . fail ( { type : ServerErrorCode . INCORRECT_FILES_IN_TORRENT , message : 'Torrents with only 1 file are supported.' } ) ; return undefined ; } return { name : extractNameFromArray ( parsedTorrent . name ) , torrentName } ; } function processMagnetURI ( body ) { const magnetUri = body . magnetUri ; const parsed = decode ( magnetUri ) ; return { name : extractNameFromArray ( parsed . name ) , magnetUri } ; } function extractNameFromArray ( name ) { return isArray ( name ) ? name [ 0 ] : name ; } async function processYoutubeSubtitles ( youtubeDL , targetUrl , videoId ) { try { const subtitles = await youtubeDL . getSubtitles ( ) ; logger . info ( 'Will create %s subtitles from youtube import %s.' , subtitles . length , targetUrl ) ; for ( const subtitle of subtitles ) { const videoCaption = new VideoCaptionModel ( { videoId , language : subtitle . language , filename : VideoCaptionModel . generateCaptionName ( subtitle . language ) } ) as MVideoCaption ; await moveAndProcessCaptionFile ( subtitle , videoCaption ) ; await sequelizeTypescript . transaction ( async ( t ) => { await VideoCaptionModel . insertOrReplaceLanguage ( videoCaption , t ) ; } ) ; } } catch ( err ) { logger . warn ( 'Cannot get video subtitles.' , { err } ) ; } }