Add project files and Docker setup

Dockerfile

@@ -0,0 +1,48 @@
+# Step 1: Use an official Python slim base image
+FROM python:3.10-slim
+
+# Step 2: Install system dependencies
+RUN apt-get update && apt-get install -y \
+    wget \
+    tar \
+    && apt-get clean
+
+# Step 3: Add a non-root user (required by Hugging Face Spaces)
+RUN useradd -m -u 1000 user
+
+# Step 4: Switch to the "user" user
+USER user
+
+# Step 5: Set home and working directory
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+
+# Step 6: Copy requirements into the container
+COPY --chown=user requirements.txt ./requirements.txt
+
+# Step 7: Install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Step 8: Copy all necessary files and folders into the container
+COPY --chown=user .output ./.output
+COPY --chown=user cache ./cache
+COPY --chown=user input ./input
+COPY --chown=user output ./output
+COPY --chown=user prompts ./prompts
+COPY --chown=user reports ./reports
+COPY --chown=user auth.py ./auth.py
+COPY --chown=user knowledge_graph.html ./knowledge_graph.html
+COPY --chown=user query_config.yaml ./query_config.yaml
+COPY --chown=user app.py ./app.py
+COPY --chown=user search_handlers.py ./search_handlers.py
+COPY --chown=user settings.yaml ./settings.yaml
+COPY --chown=user styles.css ./styles.css
+COPY --chown=user wiki.py ./wiki.py
+
+# Step 10: Expose the Streamlit default port
+EXPOSE 7860
+
+# Step 11: Define the entrypoint command
+CMD ["streamlit", "run", "app.py", "--server.port", "7860", "--server.address", "0.0.0.0"]

README.md

@@ -1,10 +1,10 @@
----
-title: PwcGraphRAG
-emoji: ⚡
-colorFrom: purple
-colorTo: yellow
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: PwcGraphRAG
+emoji: ⚡
+colorFrom: purple
+colorTo: yellow
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,305 @@
+import streamlit as st
+import asyncio
+import sys
+from pathlib import Path
+import base64
+import pandas as pd
+from typing import Literal, Tuple, Optional
+from wiki import render_wiki_tab
+from search_handlers import run_global_search, run_local_search, run_drift_search
+import auth
+
+
+import graphrag.api as api
+from graphrag.config import GraphRagConfig, load_config, resolve_paths
+from graphrag.index.create_pipeline_config import create_pipeline_config
+from graphrag.logging import PrintProgressReporter
+from graphrag.utils.storage import _create_storage, _load_table_from_storage
+
+
+st.set_page_config(page_title="GraphRAG Chat Interface", page_icon="🔍", layout="wide")
+
+# Define default avatars at the module level
+DEFAULT_USER_AVATAR = "👤"
+DEFAULT_BOT_AVATAR = "🤖"
+
+# Initialize session state for avatars
+if "user_avatar" not in st.session_state:
+    st.session_state.user_avatar = DEFAULT_USER_AVATAR
+if "bot_avatar" not in st.session_state:
+    st.session_state.bot_avatar = DEFAULT_BOT_AVATAR
+
+# Define avatar images
+USER_AVATAR = "👤"  # Default user emoji
+BOT_AVATAR = "🤖"  # Default bot emoji
+
+
+class StreamlitProgressReporter(PrintProgressReporter):
+    def __init__(self, placeholder):
+        super().__init__("")
+        self.placeholder = placeholder
+
+    def success(self, message: str):
+        self.placeholder.success(message)
+
+
+def render_chat_tab():
+    """Render the Chat tab content."""    
+    format_message_history()
+
+    # Chat input
+    if prompt := st.chat_input("Enter your query..."):
+        # Add user message to history with timestamp
+        st.session_state.messages.append(
+            {
+                "role": "user",
+                "content": prompt,
+                "timestamp": pd.Timestamp.now().strftime("%H:%M"),
+            }
+        )
+
+        # Process query
+        with st.spinner("Processing your query..."):
+            response_placeholder = st.empty()
+            try:
+                if st.session_state.search_type == "global":
+                    response, context = run_global_search(
+                        config_filepath=st.session_state.config_filepath,
+                        data_dir=st.session_state.data_dir,
+                        root_dir=st.session_state.root_dir,
+                        community_level=st.session_state.community_level,
+                        response_type=st.session_state.response_type,
+                        streaming=st.session_state.streaming,
+                        query=prompt,
+                        progress_placeholder=response_placeholder,
+                    )
+                elif st.session_state.search_type == "drift":
+                    response, context = run_drift_search(
+                        config_filepath=st.session_state.config_filepath,
+                        data_dir=st.session_state.data_dir,
+                        root_dir=st.session_state.root_dir,
+                        community_level=st.session_state.community_level,
+                        response_type=st.session_state.response_type,
+                        streaming=st.session_state.streaming,
+                        query=prompt,
+                        progress_placeholder=response_placeholder,
+                    )
+                else:
+                    response, context = run_local_search(
+                        config_filepath=st.session_state.config_filepath,
+                        data_dir=st.session_state.data_dir,
+                        root_dir=st.session_state.root_dir,
+                        community_level=st.session_state.community_level,
+                        response_type=st.session_state.response_type,
+                        streaming=st.session_state.streaming,
+                        query=prompt,
+                        progress_placeholder=response_placeholder,
+                    )
+
+                # Clear the placeholder before adding the final response
+                response_placeholder.empty()
+
+                # Add assistant response to history with timestamp
+                st.session_state.messages.append(
+                    {
+                        "role": "assistant",
+                        "content": response,
+                        "timestamp": pd.Timestamp.now().strftime("%H:%M"),
+                    }
+                )
+
+                # Show context in expander
+                with st.expander("View Search Context"):
+                    st.json(context)
+
+            except Exception as e:
+                error_message = f"Error processing query: {str(e)}"
+                st.session_state.messages.append(
+                    {
+                        "role": "assistant",
+                        "content": error_message,
+                        "timestamp": pd.Timestamp.now().strftime("%H:%M"),
+                    }
+                )
+
+        
+        st.rerun()
+
+
+def display_message(msg: str, is_user: bool = False, timestamp: str = "") -> None:
+    """Display a chat message with avatar and consistent formatting."""
+    role = "user" if is_user else "assistant"
+    message_class = "user-message" if is_user else "assistant-message"
+    avatar = st.session_state.user_avatar if is_user else st.session_state.bot_avatar
+
+    message_container = f"""
+        <div class="chat-message {message_class}">
+            <div class="avatar">
+                <div style="font-size: 25px; text-align: center;">{avatar}</div>
+            </div>
+            <div class="message-content-wrapper">
+                <div class="message-bubble">
+                    <div class="message-content">
+                        {msg}
+                    </div>
+                </div>
+                <div class="timestamp">{timestamp}</div>
+            </div>
+        </div>
+    """
+    st.markdown(message_container, unsafe_allow_html=True)
+
+
+def format_message_history() -> None:
+    """Display all messages in the chat history with consistent formatting."""
+    st.markdown('<div class="chat-container">', unsafe_allow_html=True)
+    for message in st.session_state.messages:
+        timestamp = message.get("timestamp", "")
+        display_message(
+            msg=message["content"],
+            is_user=(message["role"] == "user"),
+            timestamp=timestamp,
+        )
+    st.markdown("</div>", unsafe_allow_html=True)
+
+
+@st.cache_resource
+def load_css():
+    with open("styles.css", "r") as f:
+        return f.read()
+
+
+def initialize_session_state():
+    """Initialize session state variables if they don't exist."""
+    if "messages" not in st.session_state:
+        st.session_state.messages = []
+    if "response_placeholder" not in st.session_state:
+        st.session_state.response_placeholder = None
+    if "config_filepath" not in st.session_state:
+        st.session_state.config_filepath = None
+    if "data_dir" not in st.session_state:
+        st.session_state.data_dir = None
+    if "root_dir" not in st.session_state:
+        st.session_state.root_dir = "."
+    if "community_level" not in st.session_state:
+        st.session_state.community_level = 2
+    if "response_type" not in st.session_state:
+        st.session_state.response_type = "concise"
+    if "search_type" not in st.session_state:
+        st.session_state.search_type = "global"
+    if "streaming" not in st.session_state:
+        st.session_state.streaming = True
+    if "authenticated" not in st.session_state:
+        st.session_state.authenticated = False
+
+
+def main():
+    initialize_session_state()
+
+    # Authentication check
+    if not st.session_state.authenticated:
+        if auth.check_credentials():
+            st.session_state.authenticated = True
+            st.rerun()  # Rerun to reflect the authentication state
+        else:
+            st.stop()  # Stop further execution if authentication fails
+
+    # If authenticated, proceed with the main app
+    if st.session_state.authenticated:
+        # Main application content
+        st.title("PWC Home Assigment #1, Graphrag")
+
+        css = load_css()
+        st.markdown(f"<style>{css}</style>", unsafe_allow_html=True)
+
+        # Sidebar configuration
+        with st.sidebar:
+            # Display logos side by side at the top of the sidebar
+            col1, col2 = st.columns(2)            
+            with col1:
+                st.markdown(
+                    '<div class="logo-container"><img class="logo-image" src="https://nexttech.pwc.co.il/wp-content/uploads/2023/12/image-2.png"></div>',
+                    unsafe_allow_html=True,
+                )
+            with col2:
+                st.markdown(
+                    '<div class="logo-container"><img class="logo-image" src="https://nexttech.pwc.co.il/wp-content/uploads/2023/12/Frame.png"></div>',
+                    unsafe_allow_html=True,
+                )
+
+            st.header("Configuration")
+            st.session_state.community_level = st.number_input(
+                "Community Level",
+                min_value=0,
+                max_value=10,
+                value=st.session_state.community_level,
+                help="Controls the granularity of the search...",
+            )
+
+            # Only show response type for global and local search
+            if st.session_state.search_type != "drift":
+                st.session_state.response_type = st.selectbox(
+                    "Response Type",
+                    options=["concise", "detailed"],
+                    index=0 if st.session_state.response_type == "concise" else 1,
+                    help="Style of response generation",
+                )
+
+            st.session_state.search_type = st.selectbox(
+                "Search Type",
+                options=["global", "local", "drift"],
+                index=(
+                    0
+                    if st.session_state.search_type == "global"
+                    else 1 if st.session_state.search_type == "local" else 2
+                ),
+                help="""Search Types:
+                - Local Search: "Focuses on finding specific information by searching through direct connections in the knowledge graph. Best for precise, fact-based queries."
+                - Global Search: "Analyzes the entire document collection at a high level using community summaries. Best for understanding broad themes and general policies."
+                - DRIFT Search: "Combines local and global search capabilities, dynamically exploring connections while gathering detailed information. Best for complex queries requiring both specific details and broader context."
+                """,
+            )
+
+            # Show streaming option only for supported search types
+            if st.session_state.search_type != "drift":
+                st.session_state.streaming = st.checkbox(
+                    "Enable Streaming",
+                    value=st.session_state.streaming,
+                    help="Stream response tokens as they're generated",
+                )
+            else:
+                st.session_state.streaming = False
+                st.info("Streaming is not available for DRIFT search")
+
+            # logout button
+            if st.button("Logout"):
+                st.session_state.clear()  # Clear all session state data
+                initialize_session_state()  # Reinitialize the session state
+                st.query_params = {"restart": "true"}  # Refresh the UI
+                st.rerun()
+
+        # Create tabs
+        tab1, tab2 = st.tabs(["Assignment Documentation", "Chat"])
+
+        # readme tab content
+        with tab1:            
+            render_wiki_tab()
+
+        # Chat tab content
+        with tab2:
+            render_chat_tab()
+
+        st.sidebar.markdown(
+            """
+            <div style="position: absolute; bottom: 0; width: 100%; text-align: center; font-size: 14px; margin-bottom: -200px;">
+                Liran Baba | 
+                <a href="https://linkedin.com/in/liranba" target="_blank">LinkedIn</a> | 
+                <a href="https://huggingface.co/CordwainerSmith" target="_blank">HuggingFace</a>
+            </div>
+            """,
+            unsafe_allow_html=True,
+        )
+
+
+if __name__ == "__main__":
+    main()

auth.py

@@ -0,0 +1,39 @@
+import os
+import streamlit as st
+
+
+def check_credentials():
+    """Handles login form and returns True if authenticated successfully."""
+
+    # Check if already authenticated
+    if st.session_state.get("authenticated", False):
+        return True  # User is already authenticated
+
+    # Retrieve credentials from environment variables (set via Hugging Face Secrets)
+    expected_username = os.environ.get("APP_USERNAME")
+    expected_password = os.environ.get("APP_PASSWORD")
+
+    if not expected_username or not expected_password:
+        st.error("Server is misconfigured: missing credentials.")
+        return False
+
+    # Show the login form only if not authenticated
+    with st.form("login_form", clear_on_submit=True):
+        st.text_input("Username", key="username")
+        st.text_input("Password", type="password", key="password")
+        submit_button = st.form_submit_button("Login")
+
+        if submit_button:
+            # Validate credentials
+            if (
+                st.session_state["username"] == expected_username
+                and st.session_state["password"] == expected_password
+            ):
+                st.session_state["authenticated"] = True  # Mark user as authenticated
+                return True
+            else:
+                st.error("😕 Incorrect username or password")
+                return False  # Indicate failed authentication
+
+    # Return False if login not attempted or failed
+    return False

query_config.yaml

@@ -0,0 +1,23 @@
+drift_search:
+  max_tokens: 4000
+  drift_k_followups: 3
+  n_depth: 2
+  local_search_text_unit_prop: 0.6
+  local_search_community_prop: 0.4
+  local_search_top_k_mapped_entities: 10
+  local_search_top_k_relationships: 10
+
+local_search:
+  text_unit_prop: 0.5
+  community_prop: 0.3
+  conversation_history_max_turns: 5
+  top_k_mapped_entities: 10
+  top_k_relationships: 10
+  max_tokens: 8000
+
+global_search:
+  max_tokens: 8000
+  data_max_tokens: 8000
+  map_max_tokens: 1000
+  reduce_max_tokens: 2000
+  concurrency: 16

requirements.txt

@@ -0,0 +1,3 @@
+streamlit==1.40.1
+pandas
+graphrag==0.4.1

search_handlers.py

@@ -0,0 +1,285 @@
+import asyncio
+from pathlib import Path
+import pandas as pd
+from typing import Tuple, Optional
+from graphrag.config import GraphRagConfig, load_config, resolve_paths
+from graphrag.index.create_pipeline_config import create_pipeline_config
+from graphrag.logging import PrintProgressReporter
+from graphrag.utils.storage import _create_storage, _load_table_from_storage
+import graphrag.api as api
+
+
+class StreamlitProgressReporter(PrintProgressReporter):
+    def __init__(self, placeholder):
+        super().__init__("")
+        self.placeholder = placeholder
+
+    def success(self, message: str):
+        self.placeholder.success(message)
+
+
+def _resolve_parquet_files(
+    root_dir: str,
+    config: GraphRagConfig,
+    parquet_list: list[str],
+    optional_list: list[str],
+) -> dict[str, pd.DataFrame]:
+    """Read parquet files to a dataframe dict."""
+    dataframe_dict = {}
+    pipeline_config = create_pipeline_config(config)
+    storage_obj = _create_storage(root_dir=root_dir, config=pipeline_config.storage)
+
+    for parquet_file in parquet_list:
+        df_key = parquet_file.split(".")[0]
+        df_value = asyncio.run(
+            _load_table_from_storage(name=parquet_file, storage=storage_obj)
+        )
+        dataframe_dict[df_key] = df_value
+
+    for optional_file in optional_list:
+        file_exists = asyncio.run(storage_obj.has(optional_file))
+        df_key = optional_file.split(".")[0]
+        if file_exists:
+            df_value = asyncio.run(
+                _load_table_from_storage(name=optional_file, storage=storage_obj)
+            )
+            dataframe_dict[df_key] = df_value
+        else:
+            dataframe_dict[df_key] = None
+
+    return dataframe_dict
+
+
+def run_global_search(
+    config_filepath: Optional[str],
+    data_dir: Optional[str],
+    root_dir: str,
+    community_level: int,
+    response_type: str,
+    streaming: bool,
+    query: str,
+    progress_placeholder,
+) -> Tuple[str, dict]:
+    """Perform a global search with a given query."""
+    root = Path(root_dir).resolve()
+    config = load_config(root, config_filepath)
+    reporter = StreamlitProgressReporter(progress_placeholder)
+
+    config.storage.base_dir = data_dir or config.storage.base_dir
+    resolve_paths(config)
+
+    dataframe_dict = _resolve_parquet_files(
+        root_dir=root_dir,
+        config=config,
+        parquet_list=[
+            "create_final_nodes.parquet",
+            "create_final_entities.parquet",
+            "create_final_community_reports.parquet",
+        ],
+        optional_list=[],
+    )
+
+    final_nodes: pd.DataFrame = dataframe_dict["create_final_nodes"]
+    final_entities: pd.DataFrame = dataframe_dict["create_final_entities"]
+    final_community_reports: pd.DataFrame = dataframe_dict[
+        "create_final_community_reports"
+    ]
+
+    if streaming:
+
+        async def run_streaming_search():
+            full_response = ""
+            context_data = None
+            get_context_data = True
+            try:
+                async for stream_chunk in api.global_search_streaming(
+                    config=config,
+                    nodes=final_nodes,
+                    entities=final_entities,
+                    community_reports=final_community_reports,
+                    community_level=community_level,
+                    response_type=response_type,
+                    query=query,
+                ):
+                    if get_context_data:
+                        context_data = stream_chunk
+                        get_context_data = False
+                    else:
+                        full_response += stream_chunk
+                        progress_placeholder.markdown(full_response)
+            except Exception as e:
+                progress_placeholder.error(f"Error during streaming search: {e}")
+                return None, None
+
+            return full_response, context_data
+
+        result = asyncio.run(run_streaming_search())
+        if result is None:
+            return "", {}  # Graceful fallback
+        return result
+
+    # Non-streaming logic
+    try:
+        response, context_data = asyncio.run(
+            api.global_search(
+                config=config,
+                nodes=final_nodes,
+                entities=final_entities,
+                community_reports=final_community_reports,
+                community_level=community_level,
+                response_type=response_type,
+                query=query,
+            )
+        )
+        reporter.success(f"Global Search Response:\n{response}")
+        return response, context_data
+    except Exception as e:
+        progress_placeholder.error(f"Error during global search: {e}")
+        return "", {}  # Graceful fallback
+
+
+def run_local_search(
+    config_filepath: Optional[str],
+    data_dir: Optional[str],
+    root_dir: str,
+    community_level: int,
+    response_type: str,
+    streaming: bool,
+    query: str,
+    progress_placeholder,
+) -> Tuple[str, dict]:
+    """Perform a local search with a given query."""
+    root = Path(root_dir).resolve()
+    config = load_config(root, config_filepath)
+    reporter = StreamlitProgressReporter(progress_placeholder)
+
+    config.storage.base_dir = data_dir or config.storage.base_dir
+    resolve_paths(config)
+
+    dataframe_dict = _resolve_parquet_files(
+        root_dir=root_dir,
+        config=config,
+        parquet_list=[
+            "create_final_nodes.parquet",
+            "create_final_community_reports.parquet",
+            "create_final_text_units.parquet",
+            "create_final_relationships.parquet",
+            "create_final_entities.parquet",
+        ],
+        optional_list=["create_final_covariates.parquet"],
+    )
+
+    final_nodes: pd.DataFrame = dataframe_dict["create_final_nodes"]
+    final_community_reports: pd.DataFrame = dataframe_dict[
+        "create_final_community_reports"
+    ]
+    final_text_units: pd.DataFrame = dataframe_dict["create_final_text_units"]
+    final_relationships: pd.DataFrame = dataframe_dict["create_final_relationships"]
+    final_entities: pd.DataFrame = dataframe_dict["create_final_entities"]
+    final_covariates: Optional[pd.DataFrame] = dataframe_dict["create_final_covariates"]
+
+    if streaming:
+
+        async def run_streaming_search():
+            full_response = ""
+            context_data = None
+            get_context_data = True
+            async for stream_chunk in api.local_search_streaming(
+                config=config,
+                nodes=final_nodes,
+                entities=final_entities,
+                community_reports=final_community_reports,
+                text_units=final_text_units,
+                relationships=final_relationships,
+                covariates=final_covariates,
+                community_level=community_level,
+                response_type=response_type,
+                query=query,
+            ):
+                if get_context_data:
+                    context_data = stream_chunk
+                    get_context_data = False
+                else:
+                    full_response += stream_chunk
+                    progress_placeholder.markdown(full_response)
+            return full_response, context_data
+
+        return asyncio.run(run_streaming_search())
+
+    response, context_data = asyncio.run(
+        api.local_search(
+            config=config,
+            nodes=final_nodes,
+            entities=final_entities,
+            community_reports=final_community_reports,
+            text_units=final_text_units,
+            relationships=final_relationships,
+            covariates=final_covariates,
+            community_level=community_level,
+            response_type=response_type,
+            query=query,
+        )
+    )
+    reporter.success(f"Local Search Response:\n{response}")
+    return response, context_data
+
+
+def run_drift_search(
+    config_filepath: Optional[str],
+    data_dir: Optional[str],
+    root_dir: str,
+    community_level: int,
+    response_type: str,
+    streaming: bool,
+    query: str,
+    progress_placeholder,
+) -> Tuple[str, dict]:
+    """Perform a DRIFT search with a given query."""
+    root = Path(root_dir).resolve()
+    config = load_config(root, config_filepath)
+    reporter = StreamlitProgressReporter(progress_placeholder)
+
+    config.storage.base_dir = data_dir or config.storage.base_dir
+    resolve_paths(config)
+
+    dataframe_dict = _resolve_parquet_files(
+        root_dir=root_dir,
+        config=config,
+        parquet_list=[
+            "create_final_nodes.parquet",
+            "create_final_entities.parquet",
+            "create_final_community_reports.parquet",
+            "create_final_text_units.parquet",
+            "create_final_relationships.parquet",
+        ],
+        optional_list=[],  # Remove covariates as it's not supported
+    )
+
+    final_nodes: pd.DataFrame = dataframe_dict["create_final_nodes"]
+    final_entities: pd.DataFrame = dataframe_dict["create_final_entities"]
+    final_community_reports: pd.DataFrame = dataframe_dict[
+        "create_final_community_reports"
+    ]
+    final_text_units: pd.DataFrame = dataframe_dict["create_final_text_units"]
+    final_relationships: pd.DataFrame = dataframe_dict["create_final_relationships"]
+
+    # Note: DRIFT search doesn't support streaming
+    if streaming:
+        progress_placeholder.warning(
+            "Streaming is not supported for DRIFT search. Using standard search instead."
+        )
+
+    response, context_data = asyncio.run(
+        api.drift_search(
+            config=config,
+            nodes=final_nodes,
+            entities=final_entities,
+            community_reports=final_community_reports,
+            text_units=final_text_units,
+            relationships=final_relationships,
+            community_level=community_level,
+            query=query,
+        )
+    )
+    reporter.success(f"DRIFT Search Response:\n{response}")
+    return response, context_data

settings.yaml

@@ -0,0 +1,196 @@
+encoding_model: cl100k_base
+skip_workflows: []
+llm:
+  api_key: ${GRAPHRAG_API_KEY}
+  type: openai_chat  
+  model: gpt-4o-mini
+  model_supports_json: true
+  max_tokens: 4000
+  temperature: 0
+
+embeddings:
+  async_mode: threaded
+  batch_size: 16
+  vector_store:
+    type: lancedb
+    db_uri: 'output/lancedb'
+    container_name: default
+    overwrite: true
+  llm:
+    api_key: ${GRAPHRAG_API_KEY}
+    type: openai_embedding
+    model: text-embedding-3-small
+
+chunks:
+  size: 500
+  overlap: 50
+  group_by_columns: [id]
+
+input:
+  type: file
+  file_type: text
+  base_dir: "input"
+  file_pattern: ".*\\.txt$"
+  recursive: true
+  source_tracking: true  
+  processing_order:      
+    - path: "first_paragraphs"
+      priority: 1
+      purpose: "graph_building"
+    - path: "full_documents"
+      priority: 2
+      purpose: "retrieval"
+
+entity_extraction:
+  prompt: "prompts/entity_extraction.txt"
+  entity_types:
+    - "Baggage Type"
+    - "Dimension"
+    - "Linear Dimension"
+    - "Weight"
+    - "Material Type"
+    - "Wheel Configuration"
+    - "Measurement Unit"
+    - "Size Category"
+    - "Weight Category"
+    - "Airline"
+    - "Alliance"
+    - "Airport"
+    - "Route Type"
+    - "Travel Class"
+    - "Cabin Section"
+    - "Aircraft Type"
+    - "Restriction"
+    - "Exemption"
+    - "Policy"
+    - "Fee Structure"
+    - "Currency"
+    - "Allowance"
+    - "Special Item"
+    - "Prohibited Item"
+    - "Restricted Item"
+    - "Dangerous Good"
+    - "Fragile Item"
+    - "Valuable Item"
+    - "Required Document"
+    - "Label Type"
+    - "Tag Category"
+    - "Service Type"
+    - "Handler Role"
+    - "Service Location"
+    - "Time Period"
+    - "Passenger Type"
+    - "Membership Level"
+    - "Group Category"
+  max_gleanings: 2
+  source_filter: "first_paragraphs"
+
+claim_extraction:
+  enabled: true
+  claim_types:
+    - "Basic Size Restriction"
+    - "Oversize Condition"
+    - "Weight Limit Standard"
+    - "Overweight Condition"
+    - "Combined Dimension Limit"
+    - "Cabin Storage Requirement"
+    - "Standard Fee"
+    - "Excess Fee"
+    - "Oversize Fee"
+    - "Overweight Fee"
+    - "Special Handling Fee"
+    - "Season Surcharge"
+    - "Route-Specific Fee"
+    - "Multi-Piece Pricing"
+    - "Fee Waiver Condition"
+    - "Basic Allowance"
+    - "Class-Based Allowance"
+    - "Status-Based Allowance"
+    - "Route-Based Allowance"
+    - "Special Group Allowance"
+    - "Seasonal Allowance"
+    - "Equipment Allowance"
+    - "Prohibited Item Policy"
+    - "Restricted Item Condition"
+    - "Dangerous Goods Policy"
+    - "Special Item Restriction"
+    - "Packaging Requirement"
+    - "Declaration Requirement"
+    - "Check-in Deadline"
+    - "Special Handling Procedure"
+    - "Priority Handling Rule"
+    - "Transfer Handling Policy"
+    - "Delivery Service Policy"
+    - "Storage Policy"
+    - "Liability Limit"
+    - "Insurance Requirement"
+    - "Claim Procedure"
+    - "Compensation Policy"
+    - "Time Limit Policy"
+    - "Weather Restriction"
+    - "Seasonal Restriction"
+    - "Aircraft Limitation"
+    - "Route Restriction"
+    - "Connection Impact"
+    - "Tag Requirement"
+    - "Label Requirement"
+    - "Documentation Requirement"
+    - "Declaration Policy"
+    - "Handling Standard"
+    - "Service Level Agreement"
+    - "Priority Service Standard"
+    - "Delivery Time Standard"
+    - "Medical Exception"
+    - "Military Exception"
+    - "Diplomatic Exception"
+    - "Event Exception"
+    - "Emergency Exception"
+  prompt: "prompts/claim_extraction.txt"
+  description: "Extract baggage measurements, weight limits, and restrictions from airline documentation."
+  max_gleanings: 2
+  source_filter: "first_paragraphs"
+
+local_search:
+  text_unit_prop: 0.7
+  community_prop: 0.3
+  top_k_mapped_entities: 15
+  top_k_relationships: 15
+  max_tokens: 4000
+  source_priority:
+    graph_search: "first_paragraphs"
+    answer_retrieval: "full_documents"
+
+global_search:
+  max_tokens: 4000
+  data_max_tokens: 4000
+  map_max_tokens: 1000
+  reduce_max_tokens: 2000
+  allow_general_knowledge: false
+  min_score_threshold: 0.1
+  concurrency: 10
+
+embed_graph:
+  enabled: true
+  num_walks: 100
+  walk_length: 10
+  window_size: 5
+  iterations: 10
+
+umap:
+  enabled: true
+  n_neighbors: 15
+  min_dist: 0.1
+  n_components: 2
+
+storage:
+  type: file
+  base_dir: "output"
+
+cache:
+  type: file
+  base_dir: "cache"
+
+reporting:
+  type: file
+  base_dir: "reports"
+  include_source_tracking: true

styles.css

@@ -0,0 +1,112 @@
+/* Container for all messages */
+.chat-container {
+    display: flex;
+    flex-direction: column;
+    gap: 1rem;
+    padding: 1rem;
+}
+
+/* Message wrapper with avatar support */
+.chat-message {
+    display: flex;
+    align-items: flex-start;
+    gap: 0.5rem;
+    width: 100%;
+    max-width: 900px;
+    margin: 0.5rem 0;
+}
+
+/* Avatar container */
+.avatar {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+    overflow: hidden;
+    flex-shrink: 0;
+}
+
+.avatar img {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+}
+
+/* Message content wrapper */
+.message-content-wrapper {
+    display: flex;
+    flex-direction: column;
+    max-width: 80%;
+}
+
+/* Message bubble */
+.message-bubble {
+    padding: 1rem;
+    border-radius: 0.5rem;
+    margin: 0.2rem 0;
+}
+
+/* User message specific styling */
+.user-message {
+    flex-direction: row-reverse;
+}
+
+.user-message .message-bubble {
+    background-color: #2b313e;
+    border-top-right-radius: 0;
+    color: white;
+}
+
+/* Assistant message specific styling */
+.assistant-message .message-bubble {
+    background-color: #343741;
+    border-top-left-radius: 0;
+    color: white;
+}
+
+/* Message content */
+.message-content {
+    word-wrap: break-word;
+}
+
+/* Remove default streamlit margins */
+.stMarkdown {
+    margin: 0 !important;
+}
+
+/* Style for code blocks within messages */
+.message-content pre {
+    background-color: #1e1e1e;
+    padding: 0.5rem;
+    border-radius: 0.3rem;
+    margin: 0.5rem 0;
+    overflow-x: auto;
+}
+
+/* Improved loading spinner visibility */
+.stSpinner {
+    text-align: center;
+    margin: 1rem 0;
+}
+
+/* Time stamp styling */
+.timestamp {
+    font-size: 0.8em;
+    color: #999;
+    margin: 0.2rem 0;
+}
+
+.logo-container {
+    display: flex;
+    /* Enable flexbox layout */
+    align-items: center;
+    /* Vertically center-align items */
+    padding: 10px 0;
+    /* Add padding top/bottom */
+}
+
+.logo-image {
+    max-width: 110px;
+    /* Set maximum width */
+    height: auto;
+    /* Maintain aspect ratio */
+}

wiki.py

@@ -0,0 +1,965 @@
+# wiki.py
+# import streamlit_mermaid as stmd
+import streamlit.components.v1 as components
+import streamlit as st
+from streamlit.components.v1 import html
+
+
+def mermaid(code: str, height: int = 600) -> None:
+    components.html(
+        f"""
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <div style="height: {height}px">
+            <pre class="mermaid">
+            {code}
+            </pre>
+        </div>
+        <script type="module">
+        import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs';
+        mermaid.initialize({{ startOnLoad: true }});
+        </script>
+        """,
+        height=height,
+    )
+
+
+def render_wiki_tab():
+    """Render the Wiki tab content."""
+    st.header("Overview")   
+
+    st.markdown(
+        """
+    This documentation details the process I followed to achieve the assignment of using GraphRAG for indexing the first paragraphs of seven documents, embedding the full documents, performing initial search on the graph built from the first paragraphs, and retrieving answers from the full document content.
+    """)
+    st.markdown(
+        """
+            This project implements a specialized document processing and querying system using GraphRAG for El Al baggage requirements\allowance documentation. The system processes first paragraphs separately from full documents, enabling graph-based search while maintaining comprehensive answer retrieval capabilities.
+            
+            """
+    )
+
+    st.markdown(
+            """
+            ### Implementation Process
+
+            Initially, I attempted to implement this using separate processing paths for first paragraphs and full documents, but I discovered a more elegant solution through GraphRAG's source tracking and processing order capabilities. Instead of maintaining separate indexes, I configured a unified approach where documents were processed together but with clear priorities and purposes.
+
+            I set up the configuration to treat first paragraphs with priority 1 for graph building and full documents with priority 2 for retrieval. This was achieved through careful configuration of source tracking, processing order, and source filters in the `settings.yaml` file, which allowed me to maintain the separation of concerns.
+            """
+        )
+
+    st.markdown(
+            """
+            ### Final Implementation
+
+            The final implementation proved successful, creating a knowledge graph from the first paragraphs while maintaining access to full document content for comprehensive answers. I used entity types specific to airport security (like **Baggage Type**, **Dimension**, **Weight Limit**) and configured claim extraction to focus on relevant restrictions and allowances.
+            
+            """
+        )
+    
+    st.markdown(
+        """
+        ### Using the Chat Application
+
+        The chat application provides an interactive interface to query the GraphRAG system. Here's how it works:
+
+        ##### Getting Started:
+        - **Step 1**: Click on the chat tab.
+        - **Step 2**: Choose the desired search type from the sidebar:
+          - **Local Search**: Focuses on specific text chunks and direct relationships in the graph.
+          - **Global Search**: Analyzes the entire dataset at a high level using community summaries.
+          - **DRIFT Search**: Combines local and global search for complex queries requiring both detailed and contextual answers.
+
+        ##### Submitting a Query:
+        - Enter your question in the input field at the bottom of the chat interface.
+        - Depending on the selected search type, the system will:
+          - Use the graph for initial navigation.
+          - Retrieve answers from full documents for comprehensive responses.
+
+        ##### Viewing Results:
+        - The assistant's response appears in the chat window, formatted for clarity.        
+
+        ##### Key Features:
+        - **Streaming Responses**: Responses are displayed in real-time for supported search types.
+        - **Session History**: Previous queries and responses are retained within the session for reference.
+
+        ##### Example Queries:
+        - "What are the liquid restrictions for carry-on bags?"
+        - "How do pet carrier size restrictions compare to regular carry-on limits?"
+        """
+    )
+
+    with st.expander("Architecture", expanded=False):
+        st.markdown(
+            """            
+            The architecture of the system is designed to process data through multiple stages, including input preparation, processing, and search functionalities. Below is a detailed diagram illustrating the workflow of the system:
+            """
+        )
+
+        mermaid_code = """
+        %%{init: {'theme': 'base', 'themeVariables': {'primaryColor': '#1E90FF', 'edgeLabelBackground': '#FFFFFF', 'secondaryColor': '#F0F8FF', 'tertiaryColor': '#FFFFFF', 'primaryTextColor': '#000000'}}}%%
+        graph TD
+            subgraph Input
+                FP[First Paragraphs] --> P[Processing]
+                FD[Full Documents] --> P
+            end
+            
+            subgraph Processing
+                P --> IE[Entity Extraction]
+                P --> CD[Community Detection]
+                P --> E[Embeddings Generation]
+                
+                IE --> G[Graph Construction]
+                CD --> G
+                E --> VS[Vector Store]
+            end
+            
+            subgraph Search
+                Q[Query] --> DS[DRIFT Search]
+                DS --> GS[Graph Search]
+                DS --> FR[Full Retrieval]
+                GS --> VS
+                FR --> VS
+                GS --> A[Answer Generation]
+                FR --> A
+            end
+        """
+        mermaid(mermaid_code, height=600)
+
+    with st.expander("Graph Analysis", expanded=False):
+        st.markdown("### System Components Breakdown:")
+
+        mermaid_code = """
+        pie
+        title "System Components"
+        "Documents" : 14
+        "Text Units" : 36
+        "Entities" : 315
+        "Relationships" : 372
+        "Communities" : 66
+        """
+        mermaid(mermaid_code, height=500)
+
+        # Description and graph statistics
+        st.markdown(
+            """
+            ### Knowledge Graph Visualization
+
+            The graph displayed below represents the relationships between various entities extracted from the input data. Nodes in the graph correspond to entities like "Documents," "Policies," and "Restrictions," while edges represent the relationships or connections between these entities. The graph is constructed using the extracted entities and relationships, processed through NetworkX, and visualized with Pyvis.
+
+            **Process of Creation**:
+            - **Data Preparation**: Entities and relationships are extracted and saved as `create_final_nodes.parquet` and `create_final_relationships.parquet` files, respectively.
+            - **Graph Construction**: Using NetworkX, nodes and edges are added based on the extracted data.
+            - **Visualization**: Pyvis is used to create an interactive visualization with options like physics-based layout, node grouping, and hover effects.
+
+            The resulting graph provides insights into the data's structure, including:
+            - Node type distribution
+            - Community detection levels
+            - Connectivity patterns
+
+            Explore the graph below to understand the relationships between key entities.
+            """
+        )
+
+        # Load and display the graph visualization (HTML file)
+        with open("knowledge_graph.html", "r") as f:
+            html_content = f.read()
+        st.components.v1.html(html_content, height=800)
+
+        # Graph statistics
+        st.markdown(
+            """
+            ### Graph Statistics:
+
+            * **Number of nodes:** 427  
+            * **Number of edges:** 453  
+
+            #### Node Type Distribution:
+
+            | Node Type             | Distribution |
+            |-----------------------|--------------|
+            | REQUIRED DOCUMENT     | 39           |
+            | SERVICE TYPE          | 35           |
+            | POLICY                | 30           |
+            | RESTRICTION           | 27           |
+            | SPECIAL ITEM          | 26           |
+            | PROHIBITED ITEM       | 23           |
+            | AIRPORT               | 22           |
+            | BAGGAGE TYPE          | 21           |
+            | SERVICE LOCATION      | 18           |
+            | DANGEROUS GOOD        | 14           |
+            | ALLOWANCE             | 13           |
+            | GEO                   | 12           |
+            | MEASUREMENT UNIT      | 11           |
+            | FEE STRUCTURE         | 10           |
+            | LINEAR DIMENSION      | 8            |
+            | TIME PERIOD           | 8            |
+            | CABIN SECTION         | 8            |
+            | WEIGHT                | 8            |
+            | WEIGHT CATEGORY       | 7            |
+            | AIRLINE               | 7            |
+            | CITY                  | 7            |
+            | DIMENSION             | 6            |
+            | VALUABLE ITEM         | 5            |
+            | ROUTE TYPE            | 5            |
+            | TRAVEL CLASS          | 5            |
+            | ORGANIZATION          | 5            |
+            | PASSENGER TYPE        | 4            |
+            | RESTRICTED ITEM       | 3            |
+            | CURRENCY              | 2            |
+            | EXEMPTION             | 2            |
+            | LABEL TYPE            | 2            |
+            | MATERIAL TYPE         | 2            |
+            | CARGO                 | 2            |
+            | MEMBERSHIP LEVEL      | 2            |
+            | AIRCRAFT TYPE         | 1            |
+            | REGION                | 1            |
+            | COUNTRY               | 1            |
+            | SIZE CATEGORY         | 1            |
+            | WHEEL CONFIGURATION   | 1            |
+            | TAG CATEGORY          | 1            |
+            | GROUP CATEGORY        | 1            |
+
+            #### Most Connected Nodes:
+
+            | Node               | Connections |
+            |--------------------|-------------|
+            | EL AL              | 49          |
+            | ANIMAL             | 29          |
+            | CHECKED BAGGAGE    | 25          |
+            | BAGGAGE            | 21          |
+            | PET                | 19          |
+            """
+        )
+
+    with st.expander("Implementation Results", expanded=False):
+        st.markdown(
+          """
+          ### Document Processing
+
+          * **Total Documents**: 14 (7 first paragraphs + 7 full documents)
+          * **Text Units**: 36
+          * **Entities**: 315
+          * **Relationships**: 372
+          * **Communities**: 66 across 4 levels
+
+          ### Community Structure
+
+          * **Level 0**: 11 communities
+          * **Level 1**: 44 communities
+          * **Level 2**: 9 communities
+          * **Level 3**: 2 communities
+          """
+      )         
+
+        st.markdown("### System Operation Flow")
+
+        mermaid_code = """
+          sequenceDiagram
+              participant U as User
+              participant Q as Query Engine
+              participant G as Graph Search
+              participant V as Vector Store
+              participant D as Document Retrieval
+              
+              U->>Q: Submit Query
+              Q->>G: Search in First Paragraph Graph
+              G->>V: Lookup Relevant Entities
+              V->>D: Retrieve Full Content
+              D->>Q: Return Comprehensive Answer
+              Q->>U: Present Response
+          """
+
+        mermaid(mermaid_code, height=400)
+
+    with st.expander("Implementation Details", expanded=False):
+        st.markdown(
+            """
+            The implementation of the system follows a processing pipeline that integrates data from the first paragraphs and full documents, creating a unified structure for efficient querying. Below is the pipeline representation:
+            """
+        )
+
+        mermaid_code = """
+        flowchart TB
+            subgraph First Paragraphs
+                FP[Load First Paragraphs] --> EP[Extract Entities]
+                EP --> RP[Build Relationships]
+                RP --> CP[Create Communities]
+            end
+            
+            subgraph Full Documents
+                FD[Load Full Documents] --> CH[Chunk Documents]
+                CH --> EF[Generate Embeddings]
+            end
+            
+            subgraph Integration
+                CP --> VS[(Vector Store)]
+                EF --> VS
+            end
+            
+            subgraph Search
+                Q[Query] --> GS[Graph Search]
+                GS --> VS
+                VS --> RD[Retrieve Details]
+                RD --> AG[Answer Generation]
+            end
+        """
+        mermaid(mermaid_code, height=800)
+    with st.expander("Requirements Fulfillment", expanded=False):
+        st.markdown(
+          """
+          ### Requirements Fulfillment
+
+          **First Paragraph Processing**: ✓  
+          * Implemented through `source_filter` and `processing_order`  
+          * Verified by entity and relationship extraction  
+
+          **Full Document Embeddings**: ✓  
+          * Stored in LanceDB  
+          * Accessible for comprehensive retrieval  
+
+          **Graph-Based Search**: ✓  
+          * Communities and relationships established  
+          * DRIFT search implemented  
+
+          **Complete Answer Retrieval**: ✓  
+          * Source priority configuration  
+          * Full document content available  
+
+          ### Performance Metrics
+
+          * **Indexing Speed**: 212.44 seconds total  
+          * **Graph Density**: 372 relationships among 315 entities  
+          * **Community Structure**: 4-level hierarchy  
+          * **Vector Store Size**: 3 Lance files for different embedding types  
+          """
+      )
+
+    with st.expander("Achieving the Requirement", expanded=False):
+        st.markdown("### Source-Based Processing Control:")
+
+        st.markdown(
+          """
+          ```yaml
+          input:
+            source_tracking: true
+            processing_order:
+              - path: "first_paragraphs"
+                priority: 1
+                purpose: "graph_building"
+              - path: "full_documents"
+                priority: 2
+                purpose: "retrieval"
+          ```
+          """
+      )
+        st.markdown(
+          """
+          This configuration ensures that GraphRAG knows which content is for graph building (first paragraphs) and which is for retrieval (full documents). The priority system makes sure first paragraphs are processed first and used primarily for the knowledge graph construction.
+          """
+      )
+
+        st.markdown("### Targeted Entity and Claim Extraction:")
+
+        st.markdown(
+          """
+          ```yaml
+          entity_extraction:
+            source_filter: "first_paragraphs"
+            max_gleanings: 2
+
+          claim_extraction:
+            source_filter: "first_paragraphs"
+          ```
+          """
+      )
+        st.markdown(
+          """
+          These filters ensure that the knowledge graph (entities, relationships, and claims) is built only from the first paragraphs. This is crucial because it means our initial search will only traverse the graph built from these first paragraphs, matching the requirement. The `max_gleanings: 2` allows for thorough extraction while maintaining precision.
+          """
+      )
+
+        st.markdown("### Search Priority and Retrieval Control:")
+
+        st.markdown(
+          """
+          ```yaml
+          local_search:
+            source_priority:
+              graph_search: "first_paragraphs"
+              answer_retrieval: "full_documents"
+            text_unit_prop: 0.7
+            community_prop: 0.3
+          ```
+          """
+      )
+        st.markdown(
+          """
+          This is where the magic happens - when a query is made, the system first searches using the graph built from first paragraphs (`graph_search: "first_paragraphs"`), but when it needs to construct the answer, it pulls the content from the full documents (`answer_retrieval: "full_documents"`). 
+
+          The text_unit and community proportions ensure we're making good use of both the graph structure and the actual content. Looking at the output files we generated (`create_final_entities.parquet`, `create_final_relationships.parquet`, etc.), we can see this two-phase approach in action: the graph structure is built and stored separately from the full content, but they're linked through the unified vector store in LanceDB, allowing seamless transitions between graph search and content retrieval during query processing.
+          """
+      )
+
+    with st.expander("Improvements to Make the Graph Creation Process Leaner and Faster", expanded=False):
+        st.markdown("### Optimization of Chunk Size and Overlap:")
+
+        st.markdown(
+          """
+          ```yaml
+          chunks:
+            size: 300  # Reduced from 500
+            overlap: 25  # Reduced from 50
+            group_by_columns: [id]
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Smaller chunks with minimal overlap reduce token usage.
+          - Maintains context while processing fewer tokens per API call.
+          - Especially efficient for first paragraphs processing.
+          """
+      )
+
+        st.markdown("### Streamline Entity Types and Claims:")
+
+        st.markdown(
+          """
+          ```yaml
+          entity_extraction:
+            entity_types:
+              - "Baggage"
+              - "Restriction"
+              - "Item"
+            max_gleanings: 1  # Reduced from 2
+
+          claim_extraction:
+            enabled: false  # Disable unless absolutely necessary
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Fewer entity types mean fewer extraction operations.
+          - Single gleaning pass is often sufficient.
+          - Claims processing is expensive and often redundant.
+          """
+      )
+
+        st.markdown("### Optimize Graph Embeddings:")
+
+        st.markdown(
+          """
+          ```yaml
+          embed_graph:
+            enabled: true
+            num_walks: 50   # Reduced from 100
+            walk_length: 5  # Reduced from 10
+            window_size: 3  # Reduced from 5
+            iterations: 5   # Reduced from 10
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Fewer random walks still capture essential graph structure.
+          - Shorter walks reduce computation time.
+          - Smaller window size focuses on immediate relationships.
+          """
+      )
+
+        st.markdown("### Batch Processing and Parallelization:")
+
+        st.markdown(
+          """
+          ```yaml
+          embeddings:
+            async_mode: asyncio  # Changed from threaded
+            batch_size: 32      # Increased from 16
+            batch_max_tokens: 8191
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Asyncio performs better than threading for I/O-bound operations.
+          - Larger batch size reduces API calls.
+          - Maximizes throughput within token limits.
+          """
+      )
+
+        st.markdown("### Community Structure Optimization:")
+
+        st.markdown(
+          """
+          ```yaml
+          cluster_graph:
+            max_cluster_size: 15  # Increased slightly
+            min_cluster_size: 3   # Added parameter
+
+          community_reports:
+            max_input_length: 2000  # Reduced from default
+            max_length: 1000       # Reduced summary length
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Balanced cluster sizes reduce processing overhead.
+          - Shorter community reports still maintain essential information.
+          - Fewer tokens per report means faster processing.
+          """
+      )
+
+        st.markdown("### Caching and Storage:")
+
+        st.markdown(
+          """
+          ```yaml
+          cache:
+            type: file
+            base_dir: "cache"
+            compression: true    # Add compression
+            cache_embeddings: true
+
+          storage:
+            type: file
+            base_dir: "output"
+            compression: true    # Add compression
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Compression reduces I/O overhead.
+          - Caching embeddings prevents recomputation.
+          - File-based storage is faster than blob storage for local processing.
+          """
+      )
+
+        st.markdown("### Disable Non-Essential Features:")
+
+        st.markdown(
+          """
+          ```yaml
+          umap:
+            enabled: false  # Disable unless visualization needed
+
+          snapshots:
+            graphml: false
+            raw_entities: false
+            top_level_nodes: false
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - UMAP calculation is computationally expensive.
+          - Snapshots are useful for debugging but add overhead.
+          """
+      )
+
+        st.markdown("### LLM Configuration Optimization:")
+
+        st.markdown(
+          """
+          ```yaml
+          llm:
+            concurrent_requests: 25
+            tokens_per_minute: 150000
+            requests_per_minute: 10000
+            max_retries: 5      # Reduced from 10
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Rationale**:
+          - Balanced concurrency prevents rate limiting.
+          - Fewer retries reduce waiting time.
+          - Token and request limits prevent throttling.
+          """
+      )
+
+    with st.expander("Query Types", expanded=False):
+        st.markdown("### Local Search:")
+
+        st.markdown(
+          """
+          ```yaml
+          local_search:
+            text_unit_prop: 0.7    # Focus on specific text chunks
+            community_prop: 0.3    # Some consideration of community context
+            top_k_mapped_entities: 15
+            source_priority:
+              graph_search: "first_paragraphs"
+              answer_retrieval: "full_documents"
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Best when**: Looking for specific baggage rules or restrictions  
+          **Example Query**: "What are the liquid restrictions for carry-on bags?"  
+
+          **How it works with our data**:
+          - Searches for entities in first paragraphs (like "liquid", "carry-on").
+          - Follows direct relationships in the graph.
+          - Retrieves detailed rules from full documents.
+
+          **Meets requirement?** Yes, but in a limited way - focuses on direct connections.
+          """
+      )
+
+        st.markdown("### Global Search:")
+
+        st.markdown(
+          """
+          ```yaml
+          global_search:
+            max_tokens: 4000
+            data_max_tokens: 4000
+            min_score_threshold: 0.1
+            allow_general_knowledge: false
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Best when**: Understanding overall policies or themes  
+          **Example Query**: "What are the main types of baggage restrictions?"  
+
+          **How it works with our data**:
+          - Looks at community summaries built from first paragraphs.
+          - Provides broader context about baggage policies.
+          - Pulls supporting details from full documents.
+
+          **Meets requirement?** Partially - good for overview but might miss specific connections.
+          """
+      )
+
+        st.markdown("### DRIFT Search (Dynamic Reasoning and Inference with Flexible Traversal):")
+
+        st.markdown(
+          """
+          ```yaml
+          local_search:
+            source_priority:
+              graph_search: "first_paragraphs"
+              answer_retrieval: "full_documents"
+          ```
+          """
+      )
+        st.markdown(
+          """
+          **Best when**: Complex queries requiring both specific details and context  
+          **Example Query**: "How do pet carrier size restrictions compare to regular carry-on limits?"  
+
+          **How it works with our data**:
+          - Starts with first paragraphs graph to understand relationships between:
+            - Pet carriers
+            - Regular carry-on bags
+            - Size restrictions
+          - Uses community understanding to find related policies.
+          - Retrieves specific details from full documents.
+
+          **Meets requirement?** Yes, most comprehensively.
+          """
+      )
+
+        st.markdown("### Best Choice for Our Requirement:")
+        st.markdown(
+          """
+          **DRIFT Search** is the most suitable because:  
+          - It naturally implements our two-phase requirement:
+            - Initial search on graph (from first paragraphs).
+            - Answer retrieval from full documents.
+          - It can handle complex queries that need:
+            - Understanding of relationships (from graph).
+            - Specific details (from full documents).
+          - It can dynamically adjust between:
+            - Local search when specific rules are needed.
+            - Global search when context is important.
+          """
+      )
+    with st.expander("Configuration: full `settings.yaml`", expanded=False):
+
+        st.markdown(
+            """
+            ```yaml
+            # Root configuration for GraphRAG, a system leveraging LLMs for advanced Retrieval Augmented Generation.
+
+            encoding_model: cl100k_base
+            # Specifies the model used for token encoding. The default 'cl100k_base' is common for OpenAI's text models,
+            # determining how text is tokenized into machine-readable units.
+
+            skip_workflows: []
+            # A list of workflows to skip during execution. Empty indicates all workflows are executed.
+
+            llm:
+              api_key: ${GRAPHRAG_API_KEY}
+              # Placeholder for the API key, replaced dynamically from environment variables.
+              # Ensures secure API access for LLM queries.
+              
+              type: openai_chat
+              # Defines the type of LLM interface used. Here, it connects to OpenAI's chat-based API.
+
+              model: gpt-4o-mini
+              # Specifies the model variant to use.
+
+              model_supports_json: true
+              # Indicates whether the LLM natively supports JSON responses, useful for structured outputs.
+
+              max_tokens: 4000
+              # Maximum number of tokens in the output. Balances performance and context length.
+
+              temperature: 0
+              # Controls randomness in outputs. 0 means deterministic responses, often preferred for accuracy.
+
+            embeddings:
+              async_mode: threaded
+              # Asynchronous embedding computation mode. 'threaded' uses multi-threading for better performance.
+
+              batch_size: 16
+              # Number of data points processed per batch during embedding, balancing speed and resource use.
+
+              vector_store:
+                type: lancedb
+                # Database type used for storing vectorized embeddings. 'lancedb' supports efficient vector operations.
+
+                db_uri: 'output/lancedb'
+                # URI pointing to the database location where embeddings are stored.
+
+                container_name: default
+                # Logical name for the container storing vector data.
+
+                overwrite: true
+                # Whether to overwrite existing vectors. True allows updating the database during reruns.
+
+              llm:
+                api_key: ${GRAPHRAG_API_KEY}
+                type: openai_embedding
+                model: text-embedding-3-small
+                # Dedicated LLM for embedding tasks. A smaller, specialized model is specified for embeddings.
+
+            chunks:
+              size: 500
+              # Number of tokens per chunk of text. Controls granularity for processing long documents.
+
+              overlap: 50
+              # Overlap between adjacent chunks to ensure continuity in analysis.
+
+              group_by_columns: [id]
+              # Groups data by 'id' before chunking, preserving document boundaries.
+
+            input:
+              type: file
+              file_type: text
+              base_dir: "input"
+              file_pattern: ".*\\.txt$"
+              recursive: true
+              source_tracking: true
+              processing_order:
+                - path: "first_paragraphs"
+                  priority: 1
+                  purpose: "graph_building"
+                - path: "full_documents"
+                  priority: 2
+                  purpose: "retrieval"
+            # Specifies the data source for ingestion:
+            # - Input is file-based text.
+            # - Reads files recursively from "input" directory matching '.txt' files.
+            # - Prioritizes "first_paragraphs" for graph building and full documents for retrieval.
+
+            entity_extraction:
+              prompt: "prompts/entity_extraction.txt"
+              # Path to the custom prompt used for entity extraction tasks.
+
+              entity_types:
+                - "Baggage Type"
+                - "Dimension"
+                - "Linear Dimension"
+                - "Weight"
+                - "Material Type"
+                - "Wheel Configuration"
+                - "Measurement Unit"
+                - "Size Category"
+                - "Weight Category"
+                - "Airline"
+                - "Alliance"
+                - "Airport"
+                - "Route Type"
+                - "Travel Class"
+                - "Cabin Section"
+                - "Aircraft Type"
+                - "Restriction"
+                - "Exemption"
+                - "Policy"
+                - "Fee Structure"
+                - "Currency"
+                - "Allowance"
+                - "Special Item"
+                - "Prohibited Item"
+                - "Restricted Item"
+                - "Dangerous Good"
+                - "Fragile Item"
+                - "Valuable Item"
+                - "Required Document"
+                - "Label Type"
+                - "Tag Category"
+                - "Service Type"
+                - "Handler Role"
+                - "Service Location"
+                - "Time Period"
+                - "Passenger Type"
+                - "Membership Level"
+                - "Group Category"
+              # Defines the types of entities the system should extract.
+
+              max_gleanings: 2
+              # Maximum number of re-processing rounds to refine entity detection.
+
+              source_filter: "first_paragraphs"
+              # Restricts extraction to text from "first_paragraphs," optimizing focus.
+
+            claim_extraction:
+              enabled: true
+              # Enables claim extraction, capturing specific conditions or assertions from text.
+
+              claim_types:
+                - "Basic Size Restriction"
+                - "Oversize Condition"
+                - "Weight Limit Standard"
+                - "Overweight Condition"
+                - "Combined Dimension Limit"
+                - "Cabin Storage Requirement"
+                - "Standard Fee"
+                - "Excess Fee"
+                - "Oversize Fee"
+                - "Overweight Fee"
+                - "Special Handling Fee"
+                - "Season Surcharge"
+                - "Route-Specific Fee"
+                - "Multi-Piece Pricing"
+                - "Fee Waiver Condition"
+                - "Basic Allowance"
+                - "Class-Based Allowance"
+                - "Status-Based Allowance"
+                - "Route-Based Allowance"
+                - "Special Group Allowance"
+                - "Seasonal Allowance"
+                - "Equipment Allowance"
+                - "Prohibited Item Policy"
+                - "Restricted Item Condition"
+                - "Dangerous Goods Policy"
+                - "Special Item Restriction"
+                - "Packaging Requirement"
+                - "Declaration Requirement"
+                - "Check-in Deadline"
+                - "Special Handling Procedure"
+                - "Priority Handling Rule"
+                - "Transfer Handling Policy"
+                - "Delivery Service Policy"
+                - "Storage Policy"
+                - "Liability Limit"
+                - "Insurance Requirement"
+                - "Claim Procedure"
+                - "Compensation Policy"
+                - "Time Limit Policy"
+                - "Weather Restriction"
+                - "Seasonal Restriction"
+                - "Aircraft Limitation"
+                - "Route Restriction"
+                - "Connection Impact"
+                - "Tag Requirement"
+                - "Label Requirement"
+                - "Documentation Requirement"
+                - "Declaration Policy"
+                - "Handling Standard"
+                - "Service Level Agreement"
+                - "Priority Service Standard"
+                - "Delivery Time Standard"
+                - "Medical Exception"
+                - "Military Exception"
+                - "Diplomatic Exception"
+                - "Event Exception"
+                - "Emergency Exception"
+              # Types of claims to extract, covering diverse scenarios (e.g., fees, allowances).
+
+              prompt: "prompts/claim_extraction.txt"
+              description: "Extract baggage measurements, weight limits, and restrictions from airline documentation."
+              # Customizes the extraction logic for airline baggage policies.
+
+              max_gleanings: 2
+              source_filter: "first_paragraphs"
+              # Restricts claims to "first_paragraphs," mirroring entity extraction.
+
+            local_search:
+              text_unit_prop: 0.7
+              community_prop: 0.3
+              top_k_mapped_entities: 15
+              top_k_relationships: 15
+              max_tokens: 4000
+              source_priority:
+                graph_search: "first_paragraphs"
+                answer_retrieval: "full_documents"
+            # Configures search behavior:
+            # - Balances searches between individual text units and community-level summaries.
+            # - Limits results to top 15 entities and relationships for relevance.
+
+            global_search:
+              max_tokens: 4000
+              data_max_tokens: 4000
+              map_max_tokens: 1000
+              reduce_max_tokens: 2000
+              allow_general_knowledge: false
+              min_score_threshold: 0.1
+              concurrency: 10
+            # Defines query-wide global search capabilities:
+            # - Token limits for different operations.
+            # - Restricts non-specific general knowledge responses.
+            # - Handles up to 10 parallel queries.
+
+            embed_graph:
+              enabled: true
+              num_walks: 100
+              walk_length: 10
+              window_size: 5
+              iterations: 10
+            # Enables graph embedding (e.g., for node2vec):
+            # - Generates 100 random walks per node to learn embeddings.
+
+            umap:
+              enabled: true
+              n_neighbors: 15
+              min_dist: 0.1
+              n_components: 2
+            # Configures UMAP for dimensionality reduction and visualization.
+
+            storage:
+              type: file
+              base_dir: "output"
+            # Outputs processed data to local "output" directory.
+
+            cache:
+              type: file
+              base_dir: "cache"
+            # Stores temporary files in "cache."
+
+            reporting:
+              type: file
+              base_dir: "reports"
+              include_source_tracking: true
+            # Generates reports, including provenance for traceability.
+
+            ```
+            """
+        )