AI NPCs that can control their actions along with dialogue. For instance, if I ask an NPC to tell me its favorite magic spell, it not only tells me the spell but also performs it!
from transformers import AutoProcessor, MusicgenForConditionalGeneration
import scipy
from pydub import AudioSegment
import os

def generate_music():
    # Initialize the processor and model
    processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
    model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")

    # Prepare inputs
    inputs = processor(
        text=["Magical enchanting music pop music exciting epic adventure magical fantasy"],
        padding=True,
        return_tensors="pt",
    )

    # Generate audio
    audio_values = model.generate(**inputs, max_new_tokens=512)

    # Save as WAV file
    sampling_rate = model.config.audio_encoder.sampling_rate
    scipy.io.wavfile.write("musicgen_out.wav", rate=sampling_rate, data=audio_values[0, 0].numpy())

    # Convert WAV to MP3
    wav_audio = AudioSegment.from_wav("musicgen_out.wav")
    wav_audio.export("musicgen_out.mp3", format="mp3")

    # Delete the WAV file
    os.remove("musicgen_out.wav")
functions/generate_music_function.py
AkshitIreddy/AI-NPCs-that-can-Control-their-Actions-along-with-Dialogue
from langchain.llms import Cohere
from langchain import PromptTemplate, LLMChain
from functions.generate_music_function import generate_music
import assemblyai as aai
from dotenv import load_dotenv
import speech_recognition as sr
import requests
import shutil
import time
import json
import re
import os

load_dotenv(dotenv_path='.env')

character_name_file_path = r"Project\Content\text_files\character_identification.txt"
start_python_file_path = r"Project\Content\text_files\start_python.txt"
python_completed_file_path = r"Project\Content\text_files\python_completed_flag.txt"
output_audio_destination_path = r"Project\Content\audio\output.mp3"
output_music_destination_path = r"Project\Content\audio\music.mp3"
action_path = r"Project\Content\text_files\action.txt"

# Load data from the JSON file
with open('character_info.json', 'r') as json_file:
    character_info = json.load(json_file)

while True:

    # Check start python text file
    with open(start_python_file_path, 'r') as file:
        start = file.read()

    #if it is 0 then sleep and continue 
    if start == '0':
        time.sleep(1)
        continue

    # Set the start file back to 0
    with open(start_python_file_path, 'w') as file:
        file.write('0')

    # Capture audio from microphone
    # Initialize the Recognizer

    r = sr.Recognizer()

    # Obtain audio from the microphone
    with sr.Microphone() as source:
        print("Say something!")
        audio = r.listen(source)

    # Save the audio to a file named 'test.wav'
    with open("input.wav", "wb") as audio_file:
        audio_file.write(audio.get_wav_data())

    print("Audio saved as input.wav")

    # Speech Recognition using Assembly AI
    aai.settings.api_key = os.getenv("ASSEMBLYAI_API_KEY")
    transcriber = aai.Transcriber()
    transcript = transcriber.transcribe("./input.wav")

    player_dialogue = transcript.text
    print(player_dialogue)

    # find out who the player is speaking to
    with open(character_name_file_path, 'r') as file:
        character_name = file.read()

    bio = character_info[character_name]['bio']

    # Check if character_name exists in character_info
    if character_name in character_info:
        character_data = character_info[character_name]
        
        # Create a string for actions
        actions_string = "\n".join([f"{action}: {description}" for action, description in character_data["actions"].items()])
        
        # Create a string for talking style
        talking_style_string = "\n".join(character_data["talking_style"])

    # Load data from conversation.json
    with open('conversation.json', 'r') as json_file:
        data = json.load(json_file)

    # Extract dialogues and actions and concatenate into a paragraph
    conversation_string = ''
    for entry in data['conversations']:
        conversation_string += f"{entry['character']}: {entry['dialogue']}\n(Action: {entry['action']})\n"

    # embeddings = CohereEmbeddings(cohere_api_key=os.getenv("COHERE_API_KEY"))
    # persist_directory = 'vectordb'
    # vectordb = Chroma(persist_directory=persist_directory,
    #                     embedding_function=embeddings)
    # docs = vectordb.similarity_search(conversation_string, k=3)
    # memory_string = "\n".join(doc.page_content for doc in docs)

    # Initialise model
    llm = Cohere(cohere_api_key=os.getenv("COHERE_API_KEY"),
                    model='command', temperature=0, max_tokens=300, stop=['Hadley Smith:', "Ettore Johnson:", "Player:"])

    # Create the template string
    template = """Bio of {character_name}: \n{bio}\n\nTalking Style of {character_name}: \n{talking_style_string}\n\nActions {character_name} can do:\n{actions_string}\n\nThis is a conversation between NPCs and a Player. The NPCs must give very very small responses followed by action at all costs. The NPCs must give responses according to their talking style and must give out a "Action"(Only give action name not description) at the end of their response.\n{conversation_string}Player: {player_dialogue}\n(Action: Idle)\n{character_name}:"""

    # Create prompt
    prompt = PromptTemplate(template=template, input_variables=['conversation_string', 'bio', 'character_name', 'talking_style_string', 'player_dialogue', 'actions_string'])

    # Create and run the llm chain
    llm_chain = LLMChain(prompt=prompt, llm=llm)
    response = llm_chain.run(conversation_string=conversation_string, bio=bio, character_name=character_name, talking_style_string=talking_style_string, player_dialogue=player_dialogue, actions_string=actions_string)

    print(prompt.format_prompt(conversation_string=conversation_string, bio=bio, character_name=character_name, talking_style_string=talking_style_string, player_dialogue=player_dialogue, actions_string=actions_string).text)
    print(response)
    
    # Define a regular expression pattern to match the Action
    action_pattern = r'\(Action: (.+?)\)'

    # Use the re.search() function to find the action
    action_match = re.search(action_pattern, response)

    if action_match:
        action = action_match.group(1)
    
    response = response.replace(action_match.group(0), '').strip()

    print("Extracted Action: " + action)
    print("Extracted Response: " + response)

    # Create a new conversation entry for the player dialogue
    player_dialogue_entry = {
        "character": "Player",  
        "dialogue": player_dialogue, 
        "action": "Idle"  
    }

    # Create a new conversation entry for the response
    response_entry = {
        "character": character_name,  # The character responding
        "dialogue": response, # The response generated by the LLM
        "action": action  # The action suggested by the LLM
    }

    # Append the player's dialogue to the conversation
    data["conversations"].append(player_dialogue_entry)

    # Append the response entry to the conversation
    data["conversations"].append(response_entry)

    # Save the updated JSON data back to the file
    with open('conversation.json', 'w') as json_file:
        json.dump(data, json_file, indent=4)


    # Create Voice using Elevenlabs
    CHUNK_SIZE = 1024
    id = ""
    if character_name == "Hadley Smith":
        id = "21m00Tcm4TlvDq8ikWAM"
    elif character_name == "Ettore Johnson":
        id = "ODq5zmih8GrVes37Dizd"

    url = f"https://api.elevenlabs.io/v1/text-to-speech/{id}"

    headers = {
    "Accept": "audio/mpeg",
    "Content-Type": "application/json",
    "xi-api-key": os.getenv("ELEVENLABS_API_KEY")
    }

    data = {
    "text": response,
    "model_id": "eleven_monolingual_v1",
    "voice_settings": {
        "stability": 0.5,
        "similarity_boost": 0.5
    }
    }

    response = requests.post(url, json=data, headers=headers)
    with open('output.mp3', 'wb') as f:
        for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
            if chunk:
                f.write(chunk)

    # Copy the generated audio to the unreal engine project
    shutil.copy('output.mp3', output_audio_destination_path)
    
    # write the action to be performed
    with open(action_path, 'w') as file:
        file.write(action)
    
    if action == "Musica Harmoniosa Creatus":
        generate_music()
        # Copy the generated music to the unreal engine project
        shutil.copy('musicgen_out.mp3', output_music_destination_path)
    
    # set the complete flag to 1
    with open(python_completed_file_path, 'w') as file:
        file.write('1')
main.py
AkshitIreddy/AI-NPCs-that-can-Control-their-Actions-along-with-Dialogue