diff --git a/.gitignore b/.gitignore index ddee9b537e47686f3d80562ad0addb0832ccffce..0e62c9f245a027b4e43da2d6e0351271fef8a118 100644 --- a/.gitignore +++ b/.gitignore @@ -169,5 +169,11 @@ code/.chainlit/translations/ storage/logs/* vectorstores/* -*/.files/* +**/.files/* code/storage/models/ + +**/translations/en-US.json +**/translations/zh-CN.json + + +**/vectorstores/* \ No newline at end of file diff --git a/code/.chainlit/config.toml b/apps/ai_tutor/.chainlit/config.toml similarity index 91% rename from code/.chainlit/config.toml rename to apps/ai_tutor/.chainlit/config.toml index a76128d17ea50c55e41689ebf23f36b400567d04..eb911b503763bf984630a57d8f90771646d236df 100644 --- a/code/.chainlit/config.toml +++ b/apps/ai_tutor/.chainlit/config.toml @@ -20,10 +20,10 @@ allow_origins = ["*"] [features] # Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript) -unsafe_allow_html = true +unsafe_allow_html = false # Process and display mathematical expressions. This can clash with "$" characters in messages. -latex = true +latex = false # Automatically tag threads with the current chat profile (if a chat profile is used) auto_tag_thread = true @@ -53,7 +53,7 @@ edit_message = true [UI] # Name of the assistant. -name = "AI Tutor" +name = "Assistant" # Description of the assistant. This is used for HTML tags. # description = "" @@ -62,14 +62,14 @@ name = "AI Tutor" default_collapse_content = true # Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full". -cot = "hidden" +cot = "full" # Link to your github repo. This will add a github button in the UI's header. -github = "https://github.com/DL4DS/dl4ds_tutor" +# github = "" # Specify a CSS file that can be used to customize the user interface. # The CSS file can be served from the public directory or via an external link. -custom_css = "/public/test.css" +# custom_css = "/public/test.css" # Specify a Javascript file that can be used to customize the user interface. # The Javascript file can be served from the public directory. @@ -79,7 +79,7 @@ custom_css = "/public/test.css" # custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap" # Specify a custom meta image url. -custom_meta_image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f5/Boston_University_seal.svg/1200px-Boston_University_seal.svg.png" +# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png" # Specify a custom build directory for the frontend. # This can be used to customize the frontend code. @@ -87,7 +87,7 @@ custom_meta_image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/ # custom_build = "./public/build" [UI.theme] - default = "light" + default = "dark" #layout = "wide" #font_family = "Inter, sans-serif" # Override default MUI light theme. (Check theme.ts) diff --git a/code/app.py b/apps/ai_tutor/app.py similarity index 98% rename from code/app.py rename to apps/ai_tutor/app.py index 3b9393f4a3232a54c8f97fc834a3a85f743592f5..19ad39088ae2a78a5c4822dc0eebf12889fc74ff 100644 --- a/code/app.py +++ b/apps/ai_tutor/app.py @@ -1,3 +1,10 @@ +import sys +import os + +# Add the parent directory of dl4ds_tutor to sys.path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) + + from fastapi import FastAPI, Request, Response, HTTPException from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates diff --git a/code/modules/config/config.yml b/apps/ai_tutor/config/config.yml similarity index 100% rename from code/modules/config/config.yml rename to apps/ai_tutor/config/config.yml diff --git a/code/modules/config/project_config.yml b/apps/ai_tutor/config/project_config.yml similarity index 100% rename from code/modules/config/project_config.yml rename to apps/ai_tutor/config/project_config.yml diff --git a/code/main.py b/apps/ai_tutor/main.py similarity index 99% rename from code/main.py rename to apps/ai_tutor/main.py index e520efa9a0a2f6b85084978f4b05d1c336beefd4..ac71418f68b4cab844f16bb14cf0d9c0839479a4 100644 --- a/code/main.py +++ b/apps/ai_tutor/main.py @@ -38,7 +38,7 @@ AGENT = "Agent" YOU = "User" ERROR = "Error" -with open("modules/config/config.yml", "r") as f: +with open("../../modules/config/config.yml", "r") as f: config = yaml.safe_load(f) @@ -85,7 +85,7 @@ class Chatbot: """ Load the configuration from a YAML file. """ - with open("modules/config/config.yml", "r") as f: + with open("../../modules/config/config.yml", "r") as f: return yaml.safe_load(f) @no_type_check diff --git a/code/public/acastusphoton-svgrepo-com.svg b/apps/ai_tutor/public/acastusphoton-svgrepo-com.svg similarity index 100% rename from code/public/acastusphoton-svgrepo-com.svg rename to apps/ai_tutor/public/acastusphoton-svgrepo-com.svg diff --git a/code/public/adv-screen-recorder-svgrepo-com.svg b/apps/ai_tutor/public/adv-screen-recorder-svgrepo-com.svg similarity index 100% rename from code/public/adv-screen-recorder-svgrepo-com.svg rename to apps/ai_tutor/public/adv-screen-recorder-svgrepo-com.svg diff --git a/code/public/alarmy-svgrepo-com.svg b/apps/ai_tutor/public/alarmy-svgrepo-com.svg similarity index 100% rename from code/public/alarmy-svgrepo-com.svg rename to apps/ai_tutor/public/alarmy-svgrepo-com.svg diff --git a/code/public/avatars/ai_tutor.png b/apps/ai_tutor/public/avatars/ai_tutor.png similarity index 100% rename from code/public/avatars/ai_tutor.png rename to apps/ai_tutor/public/avatars/ai_tutor.png diff --git a/code/public/calendar-samsung-17-svgrepo-com.svg b/apps/ai_tutor/public/calendar-samsung-17-svgrepo-com.svg similarity index 100% rename from code/public/calendar-samsung-17-svgrepo-com.svg rename to apps/ai_tutor/public/calendar-samsung-17-svgrepo-com.svg diff --git a/code/public/logo_dark.png b/apps/ai_tutor/public/logo_dark.png similarity index 100% rename from code/public/logo_dark.png rename to apps/ai_tutor/public/logo_dark.png diff --git a/code/public/logo_light.png b/apps/ai_tutor/public/logo_light.png similarity index 100% rename from code/public/logo_light.png rename to apps/ai_tutor/public/logo_light.png diff --git a/code/public/space.jpg b/apps/ai_tutor/public/space.jpg similarity index 100% rename from code/public/space.jpg rename to apps/ai_tutor/public/space.jpg diff --git a/code/public/test.css b/apps/ai_tutor/public/test.css similarity index 100% rename from code/public/test.css rename to apps/ai_tutor/public/test.css diff --git a/storage/data/urls.txt b/apps/ai_tutor/storage/data/urls.txt similarity index 100% rename from storage/data/urls.txt rename to apps/ai_tutor/storage/data/urls.txt diff --git a/code/templates/cooldown.html b/apps/ai_tutor/templates/cooldown.html similarity index 100% rename from code/templates/cooldown.html rename to apps/ai_tutor/templates/cooldown.html diff --git a/code/templates/dashboard.html b/apps/ai_tutor/templates/dashboard.html similarity index 100% rename from code/templates/dashboard.html rename to apps/ai_tutor/templates/dashboard.html diff --git a/code/templates/error.html b/apps/ai_tutor/templates/error.html similarity index 100% rename from code/templates/error.html rename to apps/ai_tutor/templates/error.html diff --git a/code/templates/error_404.html b/apps/ai_tutor/templates/error_404.html similarity index 100% rename from code/templates/error_404.html rename to apps/ai_tutor/templates/error_404.html diff --git a/code/templates/login.html b/apps/ai_tutor/templates/login.html similarity index 100% rename from code/templates/login.html rename to apps/ai_tutor/templates/login.html diff --git a/code/templates/logout.html b/apps/ai_tutor/templates/logout.html similarity index 100% rename from code/templates/logout.html rename to apps/ai_tutor/templates/logout.html diff --git a/apps/chainlit_base/.chainlit/config.toml b/apps/chainlit_base/.chainlit/config.toml new file mode 100644 index 0000000000000000000000000000000000000000..eb911b503763bf984630a57d8f90771646d236df --- /dev/null +++ b/apps/chainlit_base/.chainlit/config.toml @@ -0,0 +1,120 @@ +[project] +# Whether to enable telemetry (default: true). No personal data is collected. +enable_telemetry = true + + +# List of environment variables to be provided by each user to use the app. +user_env = [] + +# Duration (in seconds) during which the session is saved when the connection is lost +session_timeout = 3600 + +# Enable third parties caching (e.g LangChain cache) +cache = false + +# Authorized origins +allow_origins = ["*"] + +# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317) +# follow_symlink = false + +[features] +# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript) +unsafe_allow_html = false + +# Process and display mathematical expressions. This can clash with "$" characters in messages. +latex = false + +# Automatically tag threads with the current chat profile (if a chat profile is used) +auto_tag_thread = true + +# Authorize users to spontaneously upload files with messages +[features.spontaneous_file_upload] + enabled = true + accept = ["*/*"] + max_files = 20 + max_size_mb = 500 + +[features.audio] + # Threshold for audio recording + min_decibels = -45 + # Delay for the user to start speaking in MS + initial_silence_timeout = 3000 + # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop. + silence_timeout = 1500 + # Above this duration (MS), the recording will forcefully stop. + max_duration = 15000 + # Duration of the audio chunks in MS + chunk_duration = 1000 + # Sample rate of the audio + sample_rate = 44100 + +edit_message = true + +[UI] +# Name of the assistant. +name = "Assistant" + +# Description of the assistant. This is used for HTML tags. +# description = "" + +# Large size content are by default collapsed for a cleaner ui +default_collapse_content = true + +# Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full". +cot = "full" + +# Link to your github repo. This will add a github button in the UI's header. +# github = "" + +# Specify a CSS file that can be used to customize the user interface. +# The CSS file can be served from the public directory or via an external link. +# custom_css = "/public/test.css" + +# Specify a Javascript file that can be used to customize the user interface. +# The Javascript file can be served from the public directory. +# custom_js = "/public/test.js" + +# Specify a custom font url. +# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap" + +# Specify a custom meta image url. +# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png" + +# Specify a custom build directory for the frontend. +# This can be used to customize the frontend code. +# Be careful: If this is a relative path, it should not start with a slash. +# custom_build = "./public/build" + +[UI.theme] + default = "dark" + #layout = "wide" + #font_family = "Inter, sans-serif" +# Override default MUI light theme. (Check theme.ts) +[UI.theme.light] + #background = "#FAFAFA" + #paper = "#FFFFFF" + + [UI.theme.light.primary] + #main = "#F80061" + #dark = "#980039" + #light = "#FFE7EB" + [UI.theme.light.text] + #primary = "#212121" + #secondary = "#616161" + +# Override default MUI dark theme. (Check theme.ts) +[UI.theme.dark] + #background = "#FAFAFA" + #paper = "#FFFFFF" + + [UI.theme.dark.primary] + #main = "#F80061" + #dark = "#980039" + #light = "#FFE7EB" + [UI.theme.dark.text] + #primary = "#EEEEEE" + #secondary = "#BDBDBD" + +[meta] +generated_by = "1.1.402" diff --git a/apps/chainlit_base/chainlit.md b/apps/chainlit_base/chainlit.md new file mode 100644 index 0000000000000000000000000000000000000000..4507ac4676a6387c4b52a0d1111e94753a102b32 --- /dev/null +++ b/apps/chainlit_base/chainlit.md @@ -0,0 +1,14 @@ +# Welcome to Chainlit! 🚀🤖 + +Hi there, Developer! 👋 We're excited to have you on board. Chainlit is a powerful tool designed to help you prototype, debug and share applications built on top of LLMs. + +## Useful Links 🔗 + +- **Documentation:** Get started with our comprehensive [Chainlit Documentation](https://docs.chainlit.io) 📚 +- **Discord Community:** Join our friendly [Chainlit Discord](https://discord.gg/k73SQ3FyUh) to ask questions, share your projects, and connect with other developers! 💬 + +We can't wait to see what you create with Chainlit! Happy coding! 💻😊 + +## Welcome screen + +To modify the welcome screen, edit the `chainlit.md` file at the root of your project. If you do not want a welcome screen, just leave this file empty. diff --git a/code/chainlit_base.py b/apps/chainlit_base/chainlit_base.py similarity index 99% rename from code/chainlit_base.py rename to apps/chainlit_base/chainlit_base.py index 82e8a1711c2615390d5d68a6ebb0d2898a1b0866..5ab87171b1862c44e1ea2810aa17211e076a2c02 100644 --- a/code/chainlit_base.py +++ b/apps/chainlit_base/chainlit_base.py @@ -22,7 +22,7 @@ AGENT = "Agent" YOU = "User" ERROR = "Error" -with open("modules/config/config.yml", "r") as f: +with open("config/config.yml", "r") as f: config = yaml.safe_load(f) @@ -51,7 +51,7 @@ class Chatbot: """ Load the configuration from a YAML file. """ - with open("modules/config/config.yml", "r") as f: + with open("config/config.yml", "r") as f: return yaml.safe_load(f) @no_type_check diff --git a/apps/chainlit_base/config/config.yml b/apps/chainlit_base/config/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..eed8ee7b9bf5e63c79f94af8116656ba48843325 --- /dev/null +++ b/apps/chainlit_base/config/config.yml @@ -0,0 +1,60 @@ +log_dir: 'storage/logs' # str +log_chunk_dir: 'storage/logs/chunks' # str +device: 'cpu' # str [cuda, cpu] + +vectorstore: + load_from_HF: True # bool + reparse_files: True # bool + data_path: 'storage/data' # str + url_file_path: 'storage/data/urls.txt' # str + expand_urls: True # bool + db_option : 'RAGatouille' # str [FAISS, Chroma, RAGatouille, RAPTOR] + db_path : 'vectorstores' # str + model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002'] + search_top_k : 3 # int + score_threshold : 0.2 # float + + faiss_params: # Not used as of now + index_path: 'vectorstores/faiss.index' # str + index_type: 'Flat' # str [Flat, HNSW, IVF] + index_dimension: 384 # int + index_nlist: 100 # int + index_nprobe: 10 # int + + colbert_params: + index_name: "new_idx" # str + +llm_params: + llm_arch: 'langchain' # [langchain] + use_history: True # bool + generate_follow_up: False # bool + memory_window: 3 # int + llm_style: 'Normal' # str [Normal, ELI5] + llm_loader: 'gpt-4o-mini' # str [local_llm, gpt-3.5-turbo-1106, gpt-4, gpt-4o-mini] + openai_params: + temperature: 0.7 # float + local_llm_params: + temperature: 0.7 # float + repo_id: 'TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF' # HuggingFace repo id + filename: 'tinyllama-1.1b-chat-v1.0.Q5_0.gguf' # Specific name of gguf file in the repo + model_path: 'storage/models/tinyllama-1.1b-chat-v1.0.Q5_0.gguf' # Path to the model file + stream: False # bool + pdf_reader: 'gpt' # str [llama, pymupdf, gpt] + +chat_logging: + log_chat: True # bool + platform: 'literalai' + callbacks: True # bool + +splitter_options: + use_splitter: True # bool + split_by_token : True # bool + remove_leftover_delimiters: True # bool + remove_chunks: False # bool + chunking_mode: 'semantic' # str [fixed, semantic] + chunk_size : 300 # int + chunk_overlap : 30 # int + chunk_separators : ["\n\n", "\n", " ", ""] # list of strings + front_chunks_to_remove : null # int or None + last_chunks_to_remove : null # int or None + delimiters_to_remove : ['\t', '\n', ' ', ' '] # list of strings diff --git a/apps/chainlit_base/config/project_config.yml b/apps/chainlit_base/config/project_config.yml new file mode 100644 index 0000000000000000000000000000000000000000..4a93a45d809ee970503b726bf4857ae5b3183c94 --- /dev/null +++ b/apps/chainlit_base/config/project_config.yml @@ -0,0 +1,7 @@ +retriever: + retriever_hf_paths: + RAGatouille: "XThomasBU/Colbert_Index" + +metadata: + metadata_links: ["https://dl4ds.github.io/sp2024/lectures/", "https://dl4ds.github.io/sp2024/schedule/"] + slide_base_link: "https://dl4ds.github.io" \ No newline at end of file diff --git a/code/chainlit.md b/code/chainlit.md deleted file mode 100644 index a0304f6b61506558ada35d2fdc78fbbc797a0674..0000000000000000000000000000000000000000 --- a/code/chainlit.md +++ /dev/null @@ -1,5 +0,0 @@ -# Welcome to DL4DS Tutor! 🚀🤖 - -Hi there, this is an LLM chatbot designed to help answer questions on the course content. - -### --- Please wait while the Tutor loads... --- diff --git a/code/modules/__init__.py b/modules/__init__.py similarity index 100% rename from code/modules/__init__.py rename to modules/__init__.py diff --git a/code/modules/chat/__init__.py b/modules/chat/__init__.py similarity index 100% rename from code/modules/chat/__init__.py rename to modules/chat/__init__.py diff --git a/code/modules/chat/base.py b/modules/chat/base.py similarity index 100% rename from code/modules/chat/base.py rename to modules/chat/base.py diff --git a/code/modules/chat/chat_model_loader.py b/modules/chat/chat_model_loader.py similarity index 100% rename from code/modules/chat/chat_model_loader.py rename to modules/chat/chat_model_loader.py diff --git a/code/modules/chat/helpers.py b/modules/chat/helpers.py similarity index 98% rename from code/modules/chat/helpers.py rename to modules/chat/helpers.py index cb140bc75e7ffb7221f26bb872359e04a2fd9096..a4ac0d1d45372c995e1a5fbe1aeb16402ce3fdb5 100644 --- a/code/modules/chat/helpers.py +++ b/modules/chat/helpers.py @@ -1,4 +1,4 @@ -from modules.config.prompts import prompts +from modules.config.prompts import prompts # TODO: MOVE THIS TO APP SPECIFIC DIRECTORY import chainlit as cl diff --git a/code/modules/chat/langchain/__init__.py b/modules/chat/langchain/__init__.py similarity index 100% rename from code/modules/chat/langchain/__init__.py rename to modules/chat/langchain/__init__.py diff --git a/code/modules/chat/langchain/langchain_rag.py b/modules/chat/langchain/langchain_rag.py similarity index 100% rename from code/modules/chat/langchain/langchain_rag.py rename to modules/chat/langchain/langchain_rag.py diff --git a/code/modules/chat/langchain/utils.py b/modules/chat/langchain/utils.py similarity index 100% rename from code/modules/chat/langchain/utils.py rename to modules/chat/langchain/utils.py diff --git a/code/modules/chat/llm_tutor.py b/modules/chat/llm_tutor.py similarity index 100% rename from code/modules/chat/llm_tutor.py rename to modules/chat/llm_tutor.py diff --git a/code/modules/chat_processor/__init__.py b/modules/chat_processor/__init__.py similarity index 100% rename from code/modules/chat_processor/__init__.py rename to modules/chat_processor/__init__.py diff --git a/code/modules/chat_processor/helpers.py b/modules/chat_processor/helpers.py similarity index 100% rename from code/modules/chat_processor/helpers.py rename to modules/chat_processor/helpers.py diff --git a/code/modules/chat_processor/literal_ai.py b/modules/chat_processor/literal_ai.py similarity index 100% rename from code/modules/chat_processor/literal_ai.py rename to modules/chat_processor/literal_ai.py diff --git a/code/modules/config/__init__.py b/modules/config/__init__.py similarity index 100% rename from code/modules/config/__init__.py rename to modules/config/__init__.py diff --git a/code/modules/dataloader/__init__.py b/modules/config/config_manager.py similarity index 100% rename from code/modules/dataloader/__init__.py rename to modules/config/config_manager.py diff --git a/code/modules/config/constants.py b/modules/config/constants.py similarity index 100% rename from code/modules/config/constants.py rename to modules/config/constants.py diff --git a/code/modules/config/prompts.py b/modules/config/prompts.py similarity index 100% rename from code/modules/config/prompts.py rename to modules/config/prompts.py diff --git a/code/modules/retriever/__init__.py b/modules/dataloader/__init__.py similarity index 100% rename from code/modules/retriever/__init__.py rename to modules/dataloader/__init__.py diff --git a/code/modules/dataloader/data_loader.py b/modules/dataloader/data_loader.py similarity index 100% rename from code/modules/dataloader/data_loader.py rename to modules/dataloader/data_loader.py diff --git a/code/modules/dataloader/helpers.py b/modules/dataloader/helpers.py similarity index 98% rename from code/modules/dataloader/helpers.py rename to modules/dataloader/helpers.py index a9b0d1e8526fdad51b2f1b16cb142f504266ccb9..20ddeb3e0fbbb2c6d1bcff86eb98e5f88bed3b59 100644 --- a/code/modules/dataloader/helpers.py +++ b/modules/dataloader/helpers.py @@ -2,7 +2,7 @@ import requests from bs4 import BeautifulSoup from urllib.parse import urlparse import tempfile -from modules.config.constants import TIMEOUT +from modules.config.constants import TIMEOUT # TODO: MOVE THIS TO APP SPECIFIC DIRECTORY def get_urls_from_file(file_path: str): diff --git a/code/modules/dataloader/pdf_readers/base.py b/modules/dataloader/pdf_readers/base.py similarity index 100% rename from code/modules/dataloader/pdf_readers/base.py rename to modules/dataloader/pdf_readers/base.py diff --git a/code/modules/dataloader/pdf_readers/gpt.py b/modules/dataloader/pdf_readers/gpt.py similarity index 100% rename from code/modules/dataloader/pdf_readers/gpt.py rename to modules/dataloader/pdf_readers/gpt.py diff --git a/code/modules/dataloader/pdf_readers/llama.py b/modules/dataloader/pdf_readers/llama.py similarity index 100% rename from code/modules/dataloader/pdf_readers/llama.py rename to modules/dataloader/pdf_readers/llama.py diff --git a/code/modules/dataloader/webpage_crawler.py b/modules/dataloader/webpage_crawler.py similarity index 100% rename from code/modules/dataloader/webpage_crawler.py rename to modules/dataloader/webpage_crawler.py diff --git a/code/modules/vectorstore/__init__.py b/modules/retriever/__init__.py similarity index 100% rename from code/modules/vectorstore/__init__.py rename to modules/retriever/__init__.py diff --git a/code/modules/retriever/base.py b/modules/retriever/base.py similarity index 100% rename from code/modules/retriever/base.py rename to modules/retriever/base.py diff --git a/code/modules/retriever/chroma_retriever.py b/modules/retriever/chroma_retriever.py similarity index 100% rename from code/modules/retriever/chroma_retriever.py rename to modules/retriever/chroma_retriever.py diff --git a/code/modules/retriever/colbert_retriever.py b/modules/retriever/colbert_retriever.py similarity index 100% rename from code/modules/retriever/colbert_retriever.py rename to modules/retriever/colbert_retriever.py diff --git a/code/modules/retriever/faiss_retriever.py b/modules/retriever/faiss_retriever.py similarity index 100% rename from code/modules/retriever/faiss_retriever.py rename to modules/retriever/faiss_retriever.py diff --git a/code/modules/retriever/helpers.py b/modules/retriever/helpers.py similarity index 100% rename from code/modules/retriever/helpers.py rename to modules/retriever/helpers.py diff --git a/code/modules/retriever/raptor_retriever.py b/modules/retriever/raptor_retriever.py similarity index 100% rename from code/modules/retriever/raptor_retriever.py rename to modules/retriever/raptor_retriever.py diff --git a/code/modules/retriever/retriever.py b/modules/retriever/retriever.py similarity index 100% rename from code/modules/retriever/retriever.py rename to modules/retriever/retriever.py diff --git a/code/modules/vectorstore/helpers.py b/modules/vectorstore/__init__.py similarity index 100% rename from code/modules/vectorstore/helpers.py rename to modules/vectorstore/__init__.py diff --git a/code/modules/vectorstore/base.py b/modules/vectorstore/base.py similarity index 100% rename from code/modules/vectorstore/base.py rename to modules/vectorstore/base.py diff --git a/code/modules/vectorstore/chroma.py b/modules/vectorstore/chroma.py similarity index 100% rename from code/modules/vectorstore/chroma.py rename to modules/vectorstore/chroma.py diff --git a/code/modules/vectorstore/colbert.py b/modules/vectorstore/colbert.py similarity index 100% rename from code/modules/vectorstore/colbert.py rename to modules/vectorstore/colbert.py diff --git a/code/modules/vectorstore/embedding_model_loader.py b/modules/vectorstore/embedding_model_loader.py similarity index 100% rename from code/modules/vectorstore/embedding_model_loader.py rename to modules/vectorstore/embedding_model_loader.py diff --git a/code/modules/vectorstore/faiss.py b/modules/vectorstore/faiss.py similarity index 100% rename from code/modules/vectorstore/faiss.py rename to modules/vectorstore/faiss.py diff --git a/modules/vectorstore/helpers.py b/modules/vectorstore/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/code/modules/vectorstore/raptor.py b/modules/vectorstore/raptor.py similarity index 100% rename from code/modules/vectorstore/raptor.py rename to modules/vectorstore/raptor.py diff --git a/code/modules/vectorstore/store_manager.py b/modules/vectorstore/store_manager.py similarity index 93% rename from code/modules/vectorstore/store_manager.py rename to modules/vectorstore/store_manager.py index 933a8878f3d237cdda52f22c7f87b5a4f6a514ae..31784feee602fec93f62328e0e8e8f7d505aafb2 100644 --- a/code/modules/vectorstore/store_manager.py +++ b/modules/vectorstore/store_manager.py @@ -164,10 +164,21 @@ class VectorStoreManager: if __name__ == "__main__": import yaml - - with open("modules/config/config.yml", "r") as f: + import argparse + + # Add argument parsing for config files + parser = argparse.ArgumentParser(description="Load configuration files.") + parser.add_argument( + "--config_file", type=str, help="Path to the main config file", required=True + ) + parser.add_argument( + "--project_config_file", type=str, help="Path to the project config file", required=True + ) + args = parser.parse_args() + + with open(args.config_file, "r") as f: config = yaml.safe_load(f) - with open("modules/config/project_config.yml", "r") as f: + with open(args.project_config_file, "r") as f: project_config = yaml.safe_load(f) # combine the two configs diff --git a/code/modules/vectorstore/vectorstore.py b/modules/vectorstore/vectorstore.py similarity index 100% rename from code/modules/vectorstore/vectorstore.py rename to modules/vectorstore/vectorstore.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..3ca6bffc7459835408e334c25031c6c56be68252 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup, find_packages + +setup( + name="modules", + version="0.1.0", + packages=find_packages(), + python_requires=">=3.7", + description="A Deep Learning for Data Science Tutor application", +) \ No newline at end of file