import { filter } from './Filter' ; import { store } from './Store' ; import { FileResult , ResultMeta } from './Store.d' ; import { form } from './Settings' ; import { ServiceProvider , VideoProvider , PostBody , PostListItem , ImageData , FileData , } from './CrawlResult' ; type Dict = { [ key in ServiceProvider ] : string ; } ; type EmbedDataArr = [ ServiceProvider | VideoProvider , string ] [ ] ; class SaveData { private readonly providerDict : Dict = { youtube : 'https://www.youtube.com/watch?v=' , fanbox : 'https://www.fanbox.cc/' , gist : 'https://gist.github.com/' , soundcloud : 'https://soundcloud.com/' , vimeo : 'https://vimeo.com/' , twitter : 'https://twitter.com/i/web/status/' , google_forms : 'https://docs.google.com/forms/d/e/' , } ; private readonly extractTextReg = new RegExp ( /<[^<>]+>/g ) ; protected readonly matchImgSrc = new RegExp ( /(?<=src=")https.*?(jpeg|jpg|png|gif|bmp)/g ) ; public receive ( data ) { this . parsePost ( data ) ; } private parsePost ( data ) { if ( data . body === null ) { return ; } const id = data . id ; const fee = data . feeRequired ; const date = data . publishedDatetime ; const check = filter . check ( { id , fee , date } ) ; if ( ! check ) { return ; } const result = { postId : data . id , type : data . type , title : data . title , date , fee , user : data . user . name , uid : data . user . userId , tags : data . tags . join ( ',' ) , files : [ ] , links : { fileId : '' , name : 'links-' + data . id , ext : 'txt' , size : null , index : 0 , text : [ ] , url : '' , retryUrl : null , } , } ; let index = 0 ; if ( form . savePostCover . checked ) { const cover = data . coverImageUrl ; if ( cover ) { const { name , ext } = this . getUrlNameAndExt ( cover ) ; const r = { fileId : this . createFileId ( ) , name , ext , size : null , index , url : cover , retryUrl : null , } ; result . files . push ( r ) ; } } if ( data . type !== 'article' ) { let text = '' ; if ( data . type === 'entry' ) { text = data . body . html . replace ( this . extractTextReg , '' ) ; } else { text = data . body . text ; } if ( text ) { const links = this . getTextLinks ( text ) ; result . links . text = result . links . text . concat ( links ) ; result . links . fileId = this . createFileId ( ) ; if ( form . saveText . checked ) { result . links . text . push ( text ) ; } } } if ( data . type === 'article' ) { let linkTexts = [ ] ; let text = '' ; for ( const block of data . body . blocks ) { if ( block . type === 'p' ) { linkTexts . push ( block . text ) ; if ( block . links && block . links . length > 0 ) { for ( const links of block . links ) { linkTexts . push ( links . url ) ; } } text += block . text + '\r\n\r\n' ; } } for ( const link of linkTexts ) { const links = this . getTextLinks ( link ) ; result . links . text = result . links . text . concat ( links ) ; result . links . fileId = this . createFileId ( ) ; } if ( form . saveText . checked ) { result . links . text . push ( text ) ; } for ( const block of data . body . blocks ) { if ( block . type === 'image' ) { const imageData = data . body . imageMap [ block . imageId ] ; if ( ! imageData ) { continue ; } index ++ ; const resource = this . getImageData ( imageData , index ) ; resource !== null && result . files . push ( resource ) ; } } for ( const block of data . body . blocks ) { if ( block . type === 'file' ) { const fileData = data . body . fileMap [ block . fileId ] ; if ( ! fileData ) { continue ; } index ++ ; const resource = this . getFileData ( fileData , index ) ; resource !== null && result . files . push ( resource ) ; } } const embedDataArr = [ ] ; for ( const [ id , embedData ] of Object . entries ( data . body . embedMap ) ) { embedDataArr . push ( [ embedData . serviceProvider , embedData . contentId ] ) ; } const embedLinks = this . getEmbedLinks ( embedDataArr , data . id ) ; result . links . text = result . links . text . concat ( embedLinks ) ; result . links . fileId = this . createFileId ( ) ; } if ( data . type === 'image' ) { for ( const imageData of data . body . images ) { if ( ! imageData ) { continue ; } index ++ ; const resource = this . getImageData ( imageData , index ) ; resource !== null && result . files . push ( resource ) ; } } if ( data . type === 'entry' ) { const imgList = data . body . html . match ( /<img.*?>/g ) ; if ( ! imgList ) { return ; } for ( const img of imgList ) { const matchUrl = img . match ( 'https.*(jpeg|jpg|png|gif|bmp)' ) ; if ( ! matchUrl ) { return ; } index ++ ; const url = matchUrl [ 0 ] ; const { name , ext } = this . getUrlNameAndExt ( url ) ; let width = 0 ; const widthMatch = img . match ( /width="(\d*?)"/ ) ; if ( widthMatch && widthMatch . length > 1 ) { width = parseInt ( widthMatch [ 1 ] ) ; } let height = 0 ; const heightMatch = img . match ( /height="(\d*?)"/ ) ; if ( heightMatch && heightMatch . length > 1 ) { height = parseInt ( heightMatch [ 1 ] ) ; } const imageData = { id : name , extension : ext , originalUrl : url , thumbnailUrl : url , width : width , height : height , } ; const resource = this . getImageData ( imageData , index ) ; resource !== null && result . files . push ( resource ) ; } } if ( data . type === 'file' ) { for ( const fileData of data . body . files ) { if ( ! fileData ) { continue ; } index ++ ; const resource = this . getFileData ( fileData , index ) ; resource !== null && result . files . push ( resource ) ; } } if ( data . type === 'video' ) { const video = data . body . video ; const embedDataArr = [ [ video . serviceProvider , video . videoId ] , ] ; const embedLinks = this . getEmbedLinks ( embedDataArr , data . id ) ; result . links . text = result . links . text . concat ( embedLinks ) ; result . links . fileId = this . createFileId ( ) ; } store . addResult ( result ) ; } private getImageData ( imageData , index ) { if ( filter . check ( { ext : imageData . extension , } ) ) { return { fileId : imageData . id , name : imageData . id , ext : imageData . extension , size : null , index , url : imageData . originalUrl , retryUrl : imageData . thumbnailUrl , } ; } return null ; } private getFileData ( fileData , index ) { if ( filter . check ( { ext : fileData . extension , } ) ) { return { fileId : fileData . id , name : fileData . name , ext : fileData . extension , size : fileData . size , index , url : fileData . url , retryUrl : null , } ; } return null ; } private getTextLinks ( text ) { const links = [ ] ; if ( ! form . saveLink . checked ) { return links ; } const textArray = text . split ( '\n' ) ; const Reg = /http[s]*:\/\/[\w=\?\.\/&\-\#\!\%]+/g ; for ( const str of textArray ) { const match = Reg . exec ( str ) ; Reg . lastIndex = 0 ; if ( match && match . length > 0 ) { for ( const link of match ) { links . push ( link ) ; } } } return links ; } private getEmbedLinks ( dataArr , postId ) { const links = [ ] ; if ( ! form . saveLink . checked ) { return links ; } for ( const data of dataArr ) { const [ serviceProvider , contentId ] = data ; let link = this . providerDict [ serviceProvider ] + contentId ; if ( serviceProvider === 'google_forms' ) { link = link + '/viewform' ; } links . push ( link ) ; } return links ; } private createFileId ( ) { return ( new Date ( ) . getTime ( ) . toString ( ) + Math . random ( ) . toString ( 16 ) . replace ( '.' , '' ) ) ; } private getUrlNameAndExt ( url ) { const split = url . split ( '/' ) ; const fileName = split [ split . length - 1 ] ; const name = fileName . split ( '.' ) [ 0 ] ; const ext = fileName . split ( '.' ) [ 1 ] ; return { name , ext , } ; } } const saveData = new SaveData ( ) ; export { saveData } ;