import { Injectable } from '@nestjs/common' ; import 'moment-timezone' ; import { CalendarComponent , CalendarResponse , fromURL , VEvent , } from 'node-ical' ; import { RRule } from 'rrule' ; import { Connection , DeepPartial } from 'typeorm' ; import { findOneIana } from 'windows-iana/dist' ; import { QueueModel } from '../queue/queue.entity' ; import { CourseModel } from './course.entity' ; import { OfficeHourModel } from './office-hour.entity' ; import moment = require ( 'moment' ) ; import { Cron } from '@nestjs/schedule' ; import { RedisService } from 'nestjs-redis' ; import * as Redlock from 'redlock' ; import { SemesterModel } from '../semester/semester.entity' ; type Moment = moment . Moment ; type CreateOfficeHour = DeepPartial < OfficeHourModel > [ ] ; const ICalStartDateMap = { Fall : 8 , Spring : 0 , Summer_1 : 4 , Summer_2 : 6 , Summer_Full : 4 , } ; const ICalEndDateMap = { Fall : 11 , Spring : 4 , Summer_1 : 6 , Summer_2 : 8 , Summer_Full : 8 , } ; @ Injectable ( ) export class IcalService { constructor ( private connection , private readonly redisService ) { } private fixOutlookTZ ( date , tz ) { const iana = findOneIana ( tz ) ; if ( iana ) { return moment ( date ) . tz ( iana , true ) ; } else { return date ; } } private rruleToDates ( rrule , eventTZ , exdateRaw ) { const { options } = rrule ; const dtstart = this . fixOutlookTZ ( moment ( options . dtstart ) , eventTZ ) ; const until = options . until && this . fixOutlookTZ ( moment ( options . until ) , eventTZ ) ; const eventTZMoment = moment . tz . zone ( findOneIana ( eventTZ ) || eventTZ ) ; const tzUTCOffsetOnDate = ( date ) => eventTZMoment . utcOffset ( date . valueOf ( ) ) ; const dtstartUTCOffset = tzUTCOffsetOnDate ( dtstart ) ; const applyOffset = ( date , utcOffset ) : Moment => moment ( date ) . subtract ( utcOffset , 'm' ) ; const preRRule = ( date ) => applyOffset ( date , dtstartUTCOffset ) ; const postRRule = ( date ) => applyOffset ( date , - dtstartUTCOffset ) ; const fixDST = ( date ) : Moment => moment ( date ) . subtract ( dtstartUTCOffset - tzUTCOffsetOnDate ( date ) , 'm' ) ; const rule = new RRule ( { freq : options . freq , interval : options . interval , wkst : options . wkst , count : options . count , byweekday : options . byweekday , dtstart : preRRule ( dtstart ) . toDate ( ) , until : until && preRRule ( until ) . toDate ( ) , } ) ; const exdates = Object . values ( exdateRaw || { } ) . map ( ( d ) => this . fixOutlookTZ ( moment ( d ) , eventTZ ) ) . map ( ( d ) => applyOffset ( d , tzUTCOffsetOnDate ( d ) ) . valueOf ( ) ) ; const in10Weeks = new Date ( dtstart . valueOf ( ) + 1000 * 60 * 60 * 24 * 7 * 10 ) ; return rule . all ( ( d ) => ! ! until || d < in10Weeks ) . filter ( ( date ) => ! exdates . includes ( date . getTime ( ) ) ) . map ( ( d ) => fixDST ( postRRule ( moment ( d ) ) ) . toDate ( ) ) ; } parseIcal ( icalData , courseId , startDate , endDate , testRegex = /\b^(Online OH)\b/ ) { const icalDataValues = Object . values ( icalData ) ; const officeHours = icalDataValues . filter ( ( iCalElement ) : iCalElement is VEvent => iCalElement . type === 'VEVENT' && iCalElement . start !== undefined && iCalElement . end !== undefined ) ; const filteredOfficeHours = officeHours . filter ( ( event ) => testRegex . test ( event . summary ) ) ; let resultOfficeHours = [ ] ; filteredOfficeHours . forEach ( ( oh ) => { const eventTZ = oh . start . tz ; const { rrule } = oh as any ; let generatedOfficeHours = [ ] ; if ( rrule ) { const duration = oh . end . getTime ( ) - oh . start . getTime ( ) ; const allDates = this . rruleToDates ( rrule , eventTZ , oh . exdate ) ; generatedOfficeHours = allDates . map ( ( date ) => ( { title : oh . summary , courseId : courseId , room : oh . location , startTime : date , endTime : new Date ( date . getTime ( ) + duration ) , } ) ) ; } else { const startTime = this . fixOutlookTZ ( moment ( oh . start ) , eventTZ ) . toDate ( ) ; generatedOfficeHours . push ( { title : oh . summary , courseId : courseId , room : oh . location , startTime : startTime , endTime : this . fixOutlookTZ ( moment ( oh . end ) , eventTZ ) . toDate ( ) , } ) ; } const filteredHours = generatedOfficeHours . filter ( ( date ) => date . startTime >= startDate && date . startTime <= endDate ) ; resultOfficeHours = resultOfficeHours . concat ( filteredHours ) ; } ) ; return resultOfficeHours ; } private getSemEnd ( sem ) { const month = ICalEndDateMap [ sem . season ] ; const year = sem . year ; return new Date ( year , month + 1 , 0 , 0 , 0 , 0 , 0 ) ; } private getSemBegin ( sem ) { const month = ICalStartDateMap [ sem . season ] ; const day = 1 ; const year = sem . year ; return new Date ( year , month , day , 0 , 0 , 0 , 0 ) ; } param course public async updateCalendarForCourse ( course ) { console . log ( `scraping ical for course "${ course . name }"(${ course . id } at url: ${ course . icalURL }...` ) ; console . time ( `scrape course ${ course . id }` ) ; let queue = await QueueModel . findOne ( { where : { courseId : course . id , room : 'Online' } , } ) ; const semester = course . semester || ( await SemesterModel . findOne ( { where : { id : course . semesterId } , } ) ) ; if ( ! queue ) { queue = await QueueModel . create ( { room : 'Online' , courseId : course . id , staffList : [ ] , questions : [ ] , allowQuestions : false , } ) . save ( ) ; } const semBegin = this . getSemBegin ( semester ) ; const semEnd = this . getSemEnd ( semester ) ; const icalURL = await fromURL ( course . icalURL ) ; const officeHours = this . parseIcal ( icalURL , course . id , semBegin , semEnd ) ; await OfficeHourModel . delete ( { courseId : course . id } ) ; await OfficeHourModel . save ( officeHours . map ( ( e ) => { e . queueId = queue . id ; return OfficeHourModel . create ( e ) ; } ) ) ; const professorHoursRegex = /\b^(Prof|Professor)/ ; const professorOfficeHours = this . parseIcal ( icalURL , course . id , semBegin , semEnd , professorHoursRegex ) ; const professorQueues = await QueueModel . find ( { where : { isProfessorQueue : true , } , } ) ; const processedProfessorOfficeHours = [ ] ; for ( const poh of professorOfficeHours ) { const professorLocation = poh . title ; if ( ! professorQueues . some ( ( q ) => q . room === professorLocation && q . courseId === course . id ) ) { const newProfQ = QueueModel . create ( { room : professorLocation , courseId : course . id , staffList : [ ] , questions : [ ] , allowQuestions : false , isProfessorQueue : true , } ) ; await newProfQ . save ( ) ; professorQueues . push ( newProfQ ) ; } const professorQueue = professorQueues . find ( ( q ) => q . room === professorLocation ) ; processedProfessorOfficeHours . push ( OfficeHourModel . create ( { queueId : professorQueue . id , ... poh , } ) ) ; } await OfficeHourModel . save ( processedProfessorOfficeHours ) ; await QueueModel . save ( professorQueues ) ; console . timeEnd ( `scrape course ${ course . id }` ) ; console . log ( 'done scraping!' ) ; } @ Cron ( '51 0 * * *' ) public async updateAllCourses ( ) { const resource = 'locks:icalcron' ; const ttl = 60000 ; const redisDB = await this . redisService . getClient ( 'db' ) ; const redlock = new Redlock ( [ redisDB ] ) ; redlock . on ( 'clientError' , function ( err ) { console . error ( 'A redis error has occurred:' , err ) ; } ) ; try { await redlock . lock ( resource , ttl ) . then ( async ( lock ) => { console . log ( 'updating course icals' ) ; const courses = await CourseModel . find ( { where : { enabled : true } , } ) ; await Promise . all ( courses . map ( ( c ) => this . updateCalendarForCourse ( c ) ) ) ; return lock . unlock ( ) . catch ( function ( err ) { console . error ( err ) ; } ) ; } ) ; } catch ( error ) { console . error ( 'A problem locking Redlock has occurred:' , error ) ; } } }