import { each , fromPairs , keyBy , isEmpty } from 'lodash' ; import { strict as assert } from 'assert' ; import type { AcademicGrp , AcademicOrg , ModuleAttributeEntry , ModuleInfo } from '../types/api' ; import type { DepartmentCodeMap , FacultyCodeMap , SemesterModule , SemesterModuleData , WritableSemesterModuleData , } from '../types/mapper' ; import type { NUSModuleAttributes , RawLesson , Semester , Workload } from '../types/modules' ; import type { CovidZoneId } from '../services/getCovidZones' ; import { Task } from '../types/tasks' ; import { Cache } from '../types/persist' ; import config from '../config' ; import BaseTask from './BaseTask' ; import GetSemesterExams from './GetSemesterExams' ; import GetSemesterTimetable from './GetSemesterTimetable' ; import GetSemesterModules from './GetSemesterModules' ; import { fromTermCode } from '../utils/api' ; import { validateSemester } from '../services/validation' ; import { removeEmptyValues , titleize , trimValues , decodeHTMLEntities } from '../utils/data' ; import { difference } from '../utils/set' ; import { Logger } from '../services/logger' ; interface Input { departments : AcademicOrg [ ] ; faculties : AcademicGrp [ ] ; } type Output = SemesterModuleData [ ] ; /**
 * Create a mapping of department code to department name from a list of faculties
 */ export const getDepartmentCodeMap = ( departments ) : DepartmentCodeMap => fromPairs ( departments . map ( ( department ) => [ department . AcademicOrganisation , department . Description ] ) ) ; export const getFacultyCodeMap = ( departments ) : FacultyCodeMap => fromPairs ( departments . map ( ( faculty ) => [ faculty . AcademicGroup , faculty . Description ] ) ) ; const attributeMap = { YEAR : 'year' , UROP : 'urop' , SSGF : 'ssgf' , SFS : 'sfs' , PRQY : 'su' , NPRY : 'su' , GRDY : 'grsu' , LABB : 'lab' , ISM : 'ism' , HFYP : 'fyp' , } ; const mpeValueMap = { S1 : [ 'mpes1' ] , S2 : [ 'mpes2' ] , 'S1&S2' : [ 'mpes1' , 'mpes2' ] , } ; export function mapAttributes ( attributes , logger ) { const nusAttributes = { } ; for ( const entry of attributes ) { if ( entry . CourseAttribute === 'MPE' ) { const mappedAttributes = mpeValueMap [ entry . CourseAttributeValue ] ; if ( ! mappedAttributes ) { logger . warn ( { value : entry . CourseAttributeValue , key : entry . CourseAttribute } , 'Non-standard course attribute value' ) ; } else { for ( const attr of mappedAttributes ) { nusAttributes [ attr ] = true ; } } continue ; } if ( ! attributeMap [ entry . CourseAttribute ] ) continue ; if ( entry . CourseAttributeValue === 'YES' || entry . CourseAttributeValue === 'HT' ) { nusAttributes [ attributeMap [ entry . CourseAttribute ] ] = true ; } else if ( entry . CourseAttributeValue !== 'NO' ) { logger . warn ( { value : entry . CourseAttributeValue , key : entry . CourseAttribute } , 'Non-standard course attribute value' ) ; } } if ( isEmpty ( nusAttributes ) ) return undefined ; return nusAttributes ; } /**
 * Clean module info
 * - Remove empty fields and fields with text like 'nil'
 * - Trim whitespace from module title, description and other text fields
 * - Properly capitalize ALL CAPS title
 * - Decode HTML entities in description such as '&224;' to 'à'
 */ export function cleanModuleInfo ( module ) { let cleanedModule = module ; if ( cleanedModule . title === cleanedModule . title . toUpperCase ( ) ) { cleanedModule . title = titleize ( cleanedModule . title ) ; } if ( cleanedModule . description != null ) { cleanedModule . description = decodeHTMLEntities ( cleanedModule . description ) ; } cleanedModule = removeEmptyValues ( cleanedModule , [ 'workload' , 'prerequisite' , 'corequisite' , 'preclusion' , ] ) ; trimValues ( cleanedModule , [ 'title' , 'description' , 'prerequisite' , 'corequisite' , 'preclusion' ] ) ; return cleanedModule ; } /**
 * Parse the workload string into a mapping of individual components to their hours.
 * If the string is unparsable, it is returned without any modification.
 */ export function parseWorkload ( workloadString ) { const cleanedWorkloadString = workloadString . replace ( /\(.*?\)/g , '' ) . replace ( /NA/gi , '0' ) . replace ( /\s+/g , '' ) ; if ( ! /^((^|-|‐)([\d.]+)){5}$/ . test ( cleanedWorkloadString ) ) return workloadString ; return cleanedWorkloadString . split ( /[-‐]/ ) . map ( ( text ) => parseFloat ( text ) ) ; } export function getLessonCovidZones ( lessons ) { const zones = new Set < CovidZoneId > ( ) ; for ( const { covidZone } of lessons ) { zones . add ( covidZone ) ; } return Array . from ( zones ) ; } /**
 * Map ModuleInfo from the API into something closer to our own representation
 */ const mapModuleInfo = ( moduleInfo , departmentMap , facultyMap , logger ) : SemesterModule => { const { Term , AcademicOrganisation , AcademicGroup , CourseTitle , WorkLoadHours , Preclusion , PreRequisite , CoRequisite , ModularCredit , Description , Subject , CatalogNumber , ModuleAttributes = [ ] , } = moduleInfo ; const [ AcadYear ] = fromTermCode ( Term ) ; return { acadYear : AcadYear , preclusion : Preclusion , description : Description , title : CourseTitle , department : departmentMap [ AcademicOrganisation . Code ] , faculty : facultyMap [ AcademicGroup . Code ] , workload : parseWorkload ( WorkLoadHours ) , prerequisite : PreRequisite , corequisite : CoRequisite , moduleCredit : ModularCredit , moduleCode : Subject + CatalogNumber , attributes : mapAttributes ( ModuleAttributes , logger ) , } ; } ; /**
 * Download, clean and combine module info, timetable, and exam info. This task
 * uses the subtasks
 * - GetSemesterExams
 * - GetSemesterModules
 * - GetSemesterTimetable
 *
 * Output:
 * - <semester>/<module code>/semesterData.json
 */ export default class GetSemesterData extends BaseTask implements Task < Input , Output > { semester : Semester ; academicYear : string ; outputCache : Cache < Output > ; get name ( ) { return `Get data for semester ${ this . semester }` ; } constructor ( semester , academicYear = config . academicYear ) { assert ( validateSemester ( semester ) , `${ semester } is not a valid semester` ) ; super ( academicYear ) ; this . semester = semester ; this . academicYear = academicYear ; this . outputCache = this . getCache < Output > ( `semester-${ semester }-module-data` ) ; this . logger = this . rootLogger . child ( { semester , year : academicYear , task : GetSemesterData . name , } ) ; } async run ( input ) { const { semester , academicYear } = this ; this . logger . info ( `Getting semester data for ${ academicYear } semester ${ semester }` ) ; const timetables = await new GetSemesterTimetable ( semester , academicYear ) . run ( ) ; const [ exams , modules ] = await Promise . all ( [ new GetSemesterExams ( semester , academicYear ) . run ( ) , new GetSemesterModules ( semester , academicYear ) . run ( input ) , ] ) ; const departmentMap = getDepartmentCodeMap ( input . departments ) ; const facultyMap = getFacultyCodeMap ( input . faculties ) ; const modulesMap = keyBy ( modules , ( moduleInfo ) => moduleInfo . Subject + moduleInfo . CatalogNumber ) ; const semesterModuleData = [ ] ; each ( modulesMap , ( moduleInfo , moduleCode ) => { const logger = this . logger . child ( { moduleCode } ) ; const rawModule = mapModuleInfo ( moduleInfo , departmentMap , facultyMap , logger ) ; const module = cleanModuleInfo ( rawModule ) ; const timetable = timetables [ moduleCode ] ; const semesterModuleDatum = { module , moduleCode , } ; if ( timetable ) { const examInfo = exams [ moduleCode ] || { } ; semesterModuleDatum . semesterData = { semester , timetable , covidZones : getLessonCovidZones ( timetable ) , ... examInfo , } ; } semesterModuleData . push ( semesterModuleDatum ) ; } ) ; const noInfoModulesWithTimetables = Array . from ( difference ( new Set ( Object . keys ( timetables ) ) , new Set ( Object . keys ( modulesMap ) ) ) ) ; if ( noInfoModulesWithTimetables . length > 0 ) { this . logger . debug ( { moduleCodes : noInfoModulesWithTimetables . sort ( ) } , 'Found modules with timetable but no info' ) ; } const noInfoModulesWithExams = Array . from ( difference ( new Set ( Object . keys ( exams ) ) , new Set ( Object . keys ( modulesMap ) ) ) ) ; if ( noInfoModulesWithExams . length > 0 ) { this . logger . debug ( { moduleCodes : noInfoModulesWithExams . sort ( ) } , 'Found modules with exam but no info' ) ; } await Promise . all ( semesterModuleData . map ( ( { moduleCode , semesterData } ) => semesterData && this . io . semesterData ( this . semester , moduleCode , semesterData ) ) ) ; await this . outputCache . write ( semesterModuleData ) ; return semesterModuleData ; } }