import { computed , observable , toJS } from 'mobx' ; import * as mm from '@magenta/music' ; import featureFlags from './feature-flags' ; import logging , { Events } from './logging' ; import { Note } from './note' ; import { NoteSequence } from './note-sequence' ; import { editor , masks , player } from '../core' ; import { Coconet } from './coconet' ; import { fromMagentaSequence , getMagentaNoteSequence } from './magenta-utils' ; import { MODEL_URL , DifferenceFromOriginal , Mood } from './constants' ; import undo , { undoable } from '../core/undo' ; interface InfillMask { step : number ; voice : number ; } export class Generator { model = new Coconet ( MODEL_URL ) ; @ observable isWorking = false ; @ observable isModelLoaded = true ; @ observable private _nSequencesToGenerate = featureFlags . baseline ? 1 : 3 ; @ computed get nSequencesToGenerate ( ) { return this . _nSequencesToGenerate ; } setNSequencesToGenerate ( nSequences ) { logging . logEvent ( Events . SET_N_SAMPLES_TO_GENERATE , nSequences ) ; this . _nSequencesToGenerate = nSequences ; } @ observable private _conventionalSurprising = 0 ; @ computed get conventionalSurprising ( ) { return this . _conventionalSurprising ; } setConventionalSurprising ( value ) { this . _conventionalSurprising = value ; } @ observable private _majorMinor = 0 ; @ computed get majorMinor ( ) { return this . _majorMinor ; } setMajorMinor ( value ) { this . _majorMinor = value ; } @ observable private _differenceFromOriginal = DifferenceFromOriginal . SomewhatDifferent ; @ computed get differenceFromOriginal ( ) { return this . _differenceFromOriginal ; } setDifferenceFromOriginal ( value ) { this . _differenceFromOriginal = value ; } @ observable candidateSequences : NoteSequence [ ] = [ ] ; @ observable selectedCandidateSequenceIndex : number | null = null ; @ computed get selectedCandidateSequence ( ) : NoteSequence | null { const index = this . selectedCandidateSequenceIndex ; if ( index === null || this . candidateSequences . length === 0 ) return null ; return this . candidateSequences [ index ] || null ; } constructor ( ) { this . loadModel ( ) ; } async loadModel ( ) { await this . model . initialize ( ) ; this . isModelLoaded = true ; } addNoteToSelected ( note ) { const sequence = this . selectedCandidateSequence ; if ( sequence ) { sequence . addNote ( note ) ; } } removeNoteFromSelected ( note ) { const sequence = this . selectedCandidateSequence ; if ( sequence ) { sequence . removeNote ( note ) ; } } setSelectedCandidateSequence ( sequence ) { const index = this . selectedCandidateSequenceIndex ; this . candidateSequences [ index ] = sequence ; } setCandidateSequences = ( sequences ) => { this . candidateSequences = sequences ; } ; getEmptySequence = ( ) => { return [ ] ; } ; selectCandidateSequence = ( index , shouldLog = true ) => { if ( shouldLog ) logging . logEvent ( Events . SELECT_CANDIDATE_SEQUENCE , index ) ; this . selectedCandidateSequenceIndex = index ; if ( player . isPlaying ) { player . restart ( ) ; } } ; @ undoable ( 'generator.commitSelectedCandidateSequence' ) commitSelectedCandidateSequence ( clearMasks = true , shouldLog = true ) { if ( shouldLog ) { logging . logEvent ( Events . CHOOSE_CANDIDATE_SEQUENCE , undo . getUndoStep ( ) ) ; } const sequence = this . selectedCandidateSequence ; if ( sequence ) { editor . addGeneratedSequence ( sequence ) ; if ( clearMasks ) masks . clearMasks ( ) ; } this . clearCandidateSequences ( false ) ; } addBackOriginalSequence = ( ) => { editor . addGeneratedSequence ( this . candidateSequences [ 0 ] ) ; this . clearCandidateSequences ( ) ; } ; clearCandidateSequences = ( shouldLog = true ) => { if ( shouldLog ) logging . logEvent ( Events . CLEAR_CANDIDATE_SEQUENCES ) ; this . selectedCandidateSequenceIndex = null ; this . candidateSequences = [ ] ; } ; getInfillMask ( ) { const infillMask = [ ] ; masks . generationMasks . forEach ( ( mask , voice ) => { mask . forEach ( maskIndex => { infillMask . push ( { voice , step : maskIndex } ) ; } ) ; } ) ; if ( infillMask . length === 0 ) return undefined ; return infillMask . sort ( ( a , b ) => { if ( a . voice === b . voice ) return a . step - b . step ; return a . voice - b . voice ; } ) ; } computeTemperature ( conventionalSurprising ) { return 2 ** conventionalSurprising ; } binHappySad ( happySad ) { if ( happySad < - 0.33 ) { return Mood . SAD ; } else if ( happySad < 0.33 ) { return Mood . NEUTRAL ; } else { return Mood . HAPPY ; } } private prepareForHarmonization ( ) { if ( this . candidateSequences . length ) { const clearMasks = false ; const shouldLog = false ; this . commitSelectedCandidateSequence ( clearMasks , shouldLog ) ; } if ( featureFlags . baseline && masks . doImplicitMasksExist ) { masks . addImplicitMasksToUserMasks ( ) ; } } private getSimilarDifferentConfig ( ) { const { differenceFromOriginal } = this ; if ( differenceFromOriginal < DifferenceFromOriginal . Similar ) { return { discourageNotes : false , nudgeFactor : 2 , } ; } else if ( differenceFromOriginal < DifferenceFromOriginal . SomewhatDifferent ) { return { discourageNotes : false , nudgeFactor : 1 , } ; } else if ( differenceFromOriginal < DifferenceFromOriginal . Different ) { return { discourageNotes : true , nudgeFactor : 1 , } ; } else if ( differenceFromOriginal <= DifferenceFromOriginal . VeryDifferent ) { return { discourageNotes : true , nudgeFactor : 2 , } ; } else { return { discourageNotes : true , nudgeFactor : 2 , } ; } } private getMoodConfig ( ) { const { majorMinor : happySad } = this ; const happyNeutralSad = this . binHappySad ( happySad ) ; let moodConfig ; if ( happyNeutralSad !== Mood . NEUTRAL ) { moodConfig = { key : editor . key , mode : editor . mode , happy : happyNeutralSad === Mood . HAPPY , } ; } return moodConfig ; } async generate ( ) { undo . beginUndoable ( 'generator.generate' ) ; this . prepareForHarmonization ( ) ; this . isWorking = true ; await delay ( 200 ) ; if ( player . isPlaying ) { player . stop ( ) ; } const { conventionalSurprising , nSequencesToGenerate } = this ; const temperature = this . computeTemperature ( conventionalSurprising ) ; const similarDifferentConfig = this . getSimilarDifferentConfig ( ) ; const moodConfig = this . getMoodConfig ( ) ; logging . logEvent ( Events . START_GENERATE , { conventionalSurprising , nSequencesToGenerate , differenceFromOriginal : this . differenceFromOriginal , majorMinor : this . majorMinor , masks : masks . generationMasks , } ) ; const outputNotes = [ ] ; for ( let i = 0 ; i < nSequencesToGenerate ; i ++ ) { const inputNotes = [ ... editor . allNotes ] ; const sequence = getMagentaNoteSequence ( inputNotes , player . bpm , editor . totalSixteenths ) ; const infillMask = this . getInfillMask ( ) ; const results = await this . model . infill ( sequence , { temperature , infillMask , similarDifferentConfig , moodConfig , } ) ; const output = fromMagentaSequence ( results ) ; const generationMasks = masks . generationMasks ; const maskIndices = new Set < string > ( ) ; generationMasks . forEach ( ( mask , voice ) => { mask . forEach ( index => maskIndices . add ( `${ voice }:${ index }` ) ) ; } ) ; const masked = output . filter ( note => { const key = `${ note . voice }:${ note . position }` ; return maskIndices . has ( key ) ; } ) ; const joined = NoteSequence . mergeConsecutiveNotes ( masked ) ; const filtered = NoteSequence . removeOverlappingNotes ( joined ) ; outputNotes . push ( filtered . notes ) ; } const maskedNotes = masks . maskedNotes ; editor . removeNotes ( maskedNotes ) ; const noteSequences = [ maskedNotes , ... outputNotes ] . map ( notes => { return new NoteSequence ( notes ) ; } ) ; logging . logEvent ( Events . GENERATE , undo . getUndoStep ( ) ) ; this . setCandidateSequences ( noteSequences ) ; this . selectCandidateSequence ( 1 , false ) ; if ( featureFlags . baseline ) { const generationMasks = toJS ( masks . generationMasks ) ; const clearMasks = true ; const shouldLog = false ; this . commitSelectedCandidateSequence ( clearMasks , shouldLog ) ; masks . setMasks ( generationMasks ) ; } this . isWorking = false ; undo . completeUndoable ( 'generator.generate' ) ; } } export default new Generator ( ) ; function delay ( ms = 1 ) { return new Promise ( resolve => setTimeout ( ( ) => resolve ( ) , ms ) ) ; }