text
Model is always disabled?
Model is always disabled?
new_line
text
#script...
#script...
new_line
text
from transformers import AutoModelForCausalLM
from transformers import AutoModelForCausalLM
new_line
text
model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2",
model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2",
new_line
text
                                             token="xxxxxx")
                                             token="xxxxxx")
new_line
new_line
text
That loads the model fine.  But if used by index returned from VectorStoreIndex for QDrant like this:
That loads the model fine.  But if used by index returned from VectorStoreIndex for QDrant like this:
new_line
new_line
text
#script...
#script...
new_line
text
query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
new_line
new_line
text
response = query_engine.query(
response = query_engine.query(
new_line
text
    "What is formula 1?"
    "What is formula 1?"
new_line
text
)
)
new_line
new_line
text
response.print_response_stream()
response.print_response_stream()
new_line
new_line
text
It errors out with a disabled error:
It errors out with a disabled error:
new_line
text
AssertionError                            Traceback (most recent call last)
AssertionError                            Traceback (most recent call last)
new_line
text
Cell In[34], line 1
Cell In[34], line 1
new_line
text
----> 1 query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
----> 1 query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
new_line
text
      3 response = query_engine.query(
      3 response = query_engine.query(
new_line
text
      4     "What is formula 1?"
      4     "What is formula 1?"
new_line
text
      5 )
      5 )
new_line
text
      7 response.print_response_stream()
      7 response.print_response_stream()
new_line
new_line
text
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/indices/base.py:376, in BaseIndex.as_query_engine(self, llm, **kwargs)
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/indices/base.py:376, in BaseIndex.as_query_engine(self, llm, **kwargs)
new_line
text
    370 from llama_index.core.query_engine.retriever_query_engine import (
    370 from llama_index.core.query_engine.retriever_query_engine import (
new_line
text
    371     RetrieverQueryEngine,
    371     RetrieverQueryEngine,
new_line
text
    372 )
    372 )
new_line
text
    374 retriever = self.as_retriever(**kwargs)
    374 retriever = self.as_retriever(**kwargs)
new_line
text
    375 llm = (
    375 llm = (
new_line
text
--> 376     resolve_llm(llm, callback_manager=self._callback_manager)
--> 376     resolve_llm(llm, callback_manager=self._callback_manager)
new_line
text
    377     if llm
    377     if llm
new_line
text
    378     else Settings.llm
    378     else Settings.llm
new_line
text
    379 )
    379 )
new_line
text
    381 return RetrieverQueryEngine.from_args(
    381 return RetrieverQueryEngine.from_args(
new_line
text
    382     retriever,
    382     retriever,
new_line
text
    383     llm=llm,
    383     llm=llm,
new_line
text
    384     **kwargs,
    384     **kwargs,
new_line
text
    385 )
    385 )
new_line
new_line
text
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/llms/utils.py:102, in resolve_llm(llm, callback_manager)
File ~/miniconda/lib/python3.9/site-packages/llama_index/core/llms/utils.py:102, in resolve_llm(llm, callback_manager)
new_line
text
     99     print("LLM is explicitly disabled. Using MockLLM.")
     99     print("LLM is explicitly disabled. Using MockLLM.")
new_line
text
    100     llm = MockLLM()
    100     llm = MockLLM()
new_line
text
--> 102 assert isinstance(llm, LLM)
--> 102 assert isinstance(llm, LLM)
new_line
text
    104 llm.callback_manager = callback_manager or Settings.callback_manager
    104 llm.callback_manager = callback_manager or Settings.callback_manager
new_line
text
    106 return llm
    106 return llm
new_line
new_line
text
AssertionError: 
AssertionError: 
new_line
new_line
text
So why is the LLM disabled?
So why is the LLM disabled?
new_line
text
Thanks!
Thanks!
Model is always disabled?
#script...
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained("distilbert/distilgpt2",
                                             token="xxxxxx")

That loads the model fine.  But if used by index returned from VectorStoreIndex for QDrant like this:

#script...
query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)

response = query_engine.query(
    "What is formula 1?"
)

response.print_response_stream()

It errors out with a disabled error:
AssertionError                            Traceback (most recent call last)
Cell In[34], line 1
----> 1 query_engine = index_from_nodes.as_query_engine(llm=model, streaming=True)
      3 response = query_engine.query(
      4     "What is formula 1?"
      5 )
      7 response.print_response_stream()

File ~/miniconda/lib/python3.9/site-packages/llama_index/core/indices/base.py:376, in BaseIndex.as_query_engine(self, llm, **kwargs)
    370 from llama_index.core.query_engine.retriever_query_engine import (
    371     RetrieverQueryEngine,
    372 )
    374 retriever = self.as_retriever(**kwargs)
    375 llm = (
--> 376     resolve_llm(llm, callback_manager=self._callback_manager)
    377     if llm
    378     else Settings.llm
    379 )
    381 return RetrieverQueryEngine.from_args(
    382     retriever,
    383     llm=llm,
    384     **kwargs,
    385 )

File ~/miniconda/lib/python3.9/site-packages/llama_index/core/llms/utils.py:102, in resolve_llm(llm, callback_manager)
     99     print("LLM is explicitly disabled. Using MockLLM.")
    100     llm = MockLLM()
--> 102 assert isinstance(llm, LLM)
    104 llm.callback_manager = callback_manager or Settings.callback_manager
    106 return llm

AssertionError: 

So why is the LLM disabled?
Thanks!
👀
John6666
https://cdn-avatars.huggingface.co/v1/production/uploads/6640bbd0220cfa8cbfdce080/wiAHUu5ewawyipNs0YFBR.png
John Smith
John6666
user