import gradio as gr import note_seq import numpy as np from transformers import AutoTokenizer, AutoModelForCausalLM # Instrument list is imported but not currently used. from constants import GM_INSTRUMENTS # Import the current midi_model tokenizer = AutoTokenizer.from_pretrained("Katpeeler/midi_model_3") model = AutoModelForCausalLM.from_pretrained("Katpeeler/midi_model_3") # Define note and bar length, relative to 120bpm. # This is overriden if the user adjusts the bpm NOTE_LENGTH_16TH_120BPM = 0.25 * 60 / 120 BAR_LENGTH_120BPM = 4.0 * 60 / 120 # Sample rate should never change, and should be imported from constants. # I will do this once I confirm I can't use a higher sample rate for playing back audio here. SAMPLE_RATE=44100 # Main method for transposing from tokens back to midi notes. # Can specify an instrument_mapper when ready to add more sounds # THIS METHOD IS FROM DR.TRISTAN BEHRENS (https://huggingface.co/TristanBehrens) def token_sequence_to_note_sequence(token_sequence, use_program=True, use_drums=True, instrument_mapper=None, only_piano=False): if isinstance(token_sequence, str): token_sequence = token_sequence.split() note_sequence = empty_note_sequence() # Render all notes. current_program = 1 current_is_drum = False current_instrument = 0 track_count = 0 for token_index, token in enumerate(token_sequence): if token == "PIECE_START": pass elif token == "PIECE_END": print("The end.") break elif token == "TRACK_START": current_bar_index = 0 track_count += 1 pass elif token == "TRACK_END": pass elif token == "KEYS_START": pass elif token == "KEYS_END": pass elif token.startswith("KEY="): pass elif token.startswith("INST"): instrument = token.split("=")[-1] if instrument != "DRUMS" and use_program: if instrument_mapper is not None: if instrument in instrument_mapper: instrument = instrument_mapper[instrument] current_program = int(instrument) current_instrument = track_count current_is_drum = False if instrument == "DRUMS" and use_drums: current_instrument = 0 current_program = 0 current_is_drum = True elif token == "BAR_START": current_time = current_bar_index * BAR_LENGTH_120BPM current_notes = {} elif token == "BAR_END": current_bar_index += 1 pass elif token.startswith("NOTE_ON"): pitch = int(token.split("=")[-1]) note = note_sequence.notes.add() note.start_time = current_time note.end_time = current_time + 4 * NOTE_LENGTH_16TH_120BPM note.pitch = pitch note.instrument = current_instrument note.program = current_program note.velocity = 80 note.is_drum = current_is_drum current_notes[pitch] = note elif token.startswith("NOTE_OFF"): pitch = int(token.split("=")[-1]) if pitch in current_notes: note = current_notes[pitch] note.end_time = current_time elif token.startswith("TIME_DELTA"): delta = float(token.split("=")[-1]) * NOTE_LENGTH_16TH_120BPM current_time += delta elif token.startswith("DENSITY="): pass elif token == "[PAD]": pass else: print(f"Ignored token {token}.") pass # Make the instruments right. instruments_drums = [] for note in note_sequence.notes: pair = [note.program, note.is_drum] if pair not in instruments_drums: instruments_drums += [pair] note.instrument = instruments_drums.index(pair) if only_piano: for note in note_sequence.notes: if not note.is_drum: note.instrument = 0 note.program = 0 return note_sequence def empty_note_sequence(qpm=120.0, total_time=0.0): note_sequence = note_seq.protobuf.music_pb2.NoteSequence() note_sequence.tempos.add().qpm = qpm note_sequence.ticks_per_quarter = note_seq.constants.STANDARD_PPQ note_sequence.total_time = total_time return note_sequence # The process that is called when the user clicks the "generate audio" button. # Currently takes in 3 number arguments, correlating to two parts of the input prompt, # and the bpm. def process(num1, num2, num3): # Prompt used to generate. I have this hard-coded currently to make generation smoother. # I include the start of the midi file, style and genre (since they are unused), start a track, # and allow the user to adjust the instrument number and the first note from the UI. created_text = f"""PIECE_START STYLE=JSFAKES GENRE=JSFAKES TRACK_START INST={num1} BAR_START NOTE_ON={num2}""" # adjustments for bpm global NOTE_LENGTH_16TH_120BPM NOTE_LENGTH_16TH_120BPM = 0.25 * 60 / num3 global BAR_LENGTH_120BPM BAR_LENGTH_120BPM = 4.0 * 60 / num3 # send the input prompt to the tokenizer, and generate input_ids = tokenizer.encode(created_text, return_tensors="pt") generated_ids = model.generate(input_ids, max_length=500) global generated_sequence generated_sequence = tokenizer.decode(generated_ids[0]) # Convert the text of notes to audio note_sequence = token_sequence_to_note_sequence(generated_sequence) # The synth engine for playing sound synth = note_seq.midi_synth.synthesize array_of_floats = synth(note_sequence, sample_rate=SAMPLE_RATE) note_plot = note_seq.plot_sequence(note_sequence, False) array_of_floats /=1.414 array_of_floats *= 32767 int16_data = array_of_floats.astype(np.int16) # return the sampmle rate and array, needed for gradio audio widget return SAMPLE_RATE, int16_data # simple call to show the generated tokens def generation(): return generated_sequence # unused call that was used to store instant feedback of the gradio sliders. # I ended up using a simpler method for them, but am keeping this in case it becomes useful later. def identity(x, state): state += 1 return x, state, state # Gradio app structure with gr.Blocks() as demo: # Title of the page gr.Markdown("Midi Generation") # The audio generation tab with gr.Tab("Audio generation"): # an audio widget audio_output = gr.Audio() # the slider widgets for the user to adjust the values for generation number1 = gr.Slider(1, 100, value=25, label="Inst number", step=1, info="Choose between 1 and 100") number2 = gr.Slider(1, 100, value=40, label="Note number", step=1, info="Choose between 1 and 100") number3 = gr.Slider(60, 140, value=120, label="BPM", step=5, info="Choose between 60 and 140") # the button to send the prompt audio_button = gr.Button("generate audio") # the token generation tab with gr.Tab("Token generation"): # a text widget to display the generated tokens text_output = gr.Textbox() # the button to display the generated tokens text_button = gr.Button("show generated tokens") # The definitions for button clicks text_button.click(generation, inputs=None, outputs=text_output) audio_button.click(process, inputs=[number1, number2, number3], outputs=audio_output) # runs the application if __name__ == "__main__": demo.launch()