import os 
import boto3
import dotenv
import json
from fastapi import FastAPI
from sqlalchemy import func, select, text
from sqlalchemy.engine import create_engine
from llama_index import SQLDatabase,ServiceContext
from llama_index.indices.struct_store import NLSQLTableQueryEngine
from llama_index.llms import OpenAI

dotenv.load_dotenv()

OPENAI_API_KEY = os.environ['OPENAI_API_KEY']
AWS_ACCESS_KEY = os.environ['AWS_ACCESS_KEY']
AWS_SECRET_KEY = os.environ['AWS_SECRET_KEY']
AWS_REGION = os.environ['AWS_REGION']
S3_STAGING_DIR = os.environ['S3_STAGING_DIR']
DATABASE = os.environ['DATABASE']
WORKGROUP = os.environ['WORKGROUP']
TABLE = os.environ['TABLE']

app = FastAPI()
@app.get("/helloworld")
async def helloworld():
    return {"message": "Hello World"}

@app.get("/query")
async def query(query_str : str = 'Write SQL in PrestoSQL format. Which month (order_purchase_timestamp) have the most order ?'):
    
    boto3.client(
        'athena',
        aws_access_key_id=AWS_ACCESS_KEY,
        aws_secret_access_key=AWS_SECRET_KEY,
        region_name=AWS_REGION,
    )

    llm = OpenAI(model="gpt-4",temperature=0, max_tokens=1024)

    conn_str = "awsathena+rest://:@athena.{region_name}.amazonaws.com:443/"\
            "{database}?s3_staging_dir={s3_staging_dir}?work_group={workgroup}"

    engine = create_engine(conn_str.format(
        region_name=AWS_REGION,
        s3_staging_dir=S3_STAGING_DIR,
        database=DATABASE,
        workgroup=WORKGROUP
    ))
    service_context = ServiceContext.from_defaults(
        llm=llm
    )
    sql_database = SQLDatabase(engine, include_tables=[TABLE])

    query_engine = NLSQLTableQueryEngine(
        sql_database=sql_database,
        tables=[TABLE],
        service_context=service_context
    )
    query_prompt = (query_str)
    response = query_engine.query(query_prompt)

    result = {
        "result": response.response,
        "SQL Query": response.metadata['sql_query']
    }
    
    return json.dumps(result)
aws-example/text-to-sql-openai/app.py
vultureprime/deploy-ai-model
from beam import App, Runtime, Image, Output, Volume, VolumeType
import time
import torch
from transformers import (
    GenerationConfig,
    LlamaForCausalLM,
    LlamaTokenizer,
)
from peft import PeftModel
import transformers
import json
import os.path as osp
from typing import Union

base_model = "ChanonUtupon/openthaigpt-merge-lora-llama-2-7B-3470k"
lora_weights = 'openthaigpt/openthaigpt-1.0.0-beta-7b-chat'


class Prompter(object):
    __slots__ = ("template", "_verbose")

    def __init__(self, template_name: str = "", verbose: bool = False):
        self._verbose = verbose
        template_name = "alpaca"
        self.template = {
            "description": "Template used by Alpaca-LoRA.",
            "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
            "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n",
            "response_split": "### Response:"
        }
        if self._verbose:
            print(
                f"Using prompt template {template_name}: {self.template['description']}"
            )

    def generate_prompt(
        self,
        instruction: str,
        input: Union[None, str] = None,
        label: Union[None, str] = None,
    ) -> str:
        # returns the full prompt from instruction and optional input
        # if a label (=response, =output) is provided, it's also appended.
        if input:
            res = self.template["prompt_input"].format(
                instruction=instruction, input=input
            )
        else:
            res = self.template["prompt_no_input"].format(
                instruction=instruction
            )
        if label:
            res = f"{res}{label}"
        if self._verbose:
            print(res)
        return res

    def get_response(self, output: str) -> str:
        return output.split(self.template["response_split"])[1].strip()


app = App(
    name="llama2",
    runtime=Runtime(
        cpu=4,
        memory="32Gi",
        gpu="A10G",
        image=Image(
            python_packages=[
                "accelerate",
                "transformers",
                "torch",
                "sentencepiece",
                "peft"
            ],
        ),
    ),
    volumes=[
        Volume(
            name="llama2-openthaigpt",
            path="./llama2-openthaigpt",
            volume_type=VolumeType.Persistent,
        )
    ],
)

def load_models():
    tokenizer = LlamaTokenizer.from_pretrained(
            base_model,
            cache_dir="./llama2-openthaigpt",
    )
    model = LlamaForCausalLM.from_pretrained(
        base_model,
        load_in_8bit=False,
        cache_dir="./llama2-openthaigpt",
        torch_dtype=torch.float16,
        device_map="auto",
    )

    model = PeftModel.from_pretrained(
        model,
        lora_weights,
        cache_dir="./llama2-openthaigpt",
        torch_dtype=torch.float16,
        device_map="auto",
    )

    return tokenizer, model


@app.rest_api(loader=load_models)
def generate(**inputs):
    prompt_template = ""
    prompter = Prompter(prompt_template)

    tokenizer, model = inputs["context"]
    # prompt = inputs["prompt"]
    text_input = inputs["prompt"]

    prompt = prompter.generate_prompt(text_input, None)

    load_8bit = False
    tokenizer.pad_token_id = 0  # unk
    tokenizer.bos_token_id = 1
    tokenizer.eos_token_id = 2

    if not load_8bit:
        model.half()  # seems to fix bugs for some users.

    model.eval()
    if torch.__version__ >= "2":
        model = torch.compile(model)

    inputs = tokenizer(prompt, return_tensors="pt")
    input_ids = inputs["input_ids"].to("cuda")

    generation_config = GenerationConfig(
        temperature=0.1,
        top_p=0.75,
        top_k=40,
        num_beams=4,
    )
    start = time.time()
    with torch.no_grad():
        generation_output = model.generate(
            input_ids=input_ids,
            generation_config=generation_config,
            return_dict_in_generate=True,
            output_scores=True,
            max_new_tokens=128,
            early_stopping=True,
            repetition_penalty=1,
            no_repeat_ngram_size=0
        )

    s = generation_output.sequences[0]
    decoded_output = tokenizer.decode(s).split("### Response:")[1].strip()
    return {"pred": decoded_output}
beam-cloud-exmaple/openthaigpt-1.0.0-beta-pytorch/app.py
vultureprime/deploy-ai-model
from transformers import AutoTokenizer, AutoModelForCausalLM
import transformers
import torch
import time
print(transformers.__version__,torch.__version__)

model_id = "tiiuae/falcon-40b-instruct"

tokenizer = AutoTokenizer.from_pretrained(
    model_id,
    cache_dir="./falcon-40b",

)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
    trust_remote_code=True,
    load_in_8bit=True,
    device_map="auto",
    cache_dir="./falcon-40b",
)

pipeline = transformers.pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
)

start = time.time()
sequences = pipeline(
   "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:",
    max_length=200,
    do_sample=True,
    top_k=10,
    num_return_sequences=1,
    eos_token_id=tokenizer.eos_token_id,
)
print(time.time() - start )
for seq in sequences:
    print(f"Result: {seq['generated_text']}")
paperspace-example/falcon-40b-instruct-8bit-pytorch/app.py
vultureprime/deploy-ai-model
from langchain.document_loaders import WebBaseLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma
from langchain.chat_models import ChatOpenAI
from langchain.chains import RetrievalQAWithSourcesChain
from fastapi import FastAPI
import dotenv

dotenv.load_dotenv()

app = FastAPI()
@app.get("/helloworld")
async def helloworld():
    return {"message": "Hello World"}

@app.get("/loadAndStore")
async def load_and_store(url : str = 'https://lilianweng.github.io/posts/2023-06-23-agent', chunk_size : int = 1024, chunk_overlap : int = 0, collection_name : str = 'temp1'):
    loader = WebBaseLoader(url) 
    data = loader.load()
    text_splitter = RecursiveCharacterTextSplitter(chunk_size = chunk_size, chunk_overlap = chunk_overlap) 
    all_splits = text_splitter.split_documents(data) 
    Chroma.from_documents(collection_name=collection_name,documents=all_splits, embedding=OpenAIEmbeddings(),persist_directory='./chroma_db') 
    return {"Message": "Completed"}

@app.get("/peekDocument")
async def peek_document(collection_name : str = 'temp1'):
    vectorstore = Chroma(collection_name=collection_name,persist_directory='./chroma_db',embedding_function = OpenAIEmbeddings()) #Load collection from disk.
    return {"Top documnet in collection": vectorstore._collection.peek()} #Return top document in collection.

@app.get("/search")
async def search(query : str = 'What are the approaches to Task Decomposition?', collection_name : str = 'temp1'):
    vectorstore = Chroma(collection_name=collection_name,persist_directory='./chroma_db',embedding_function = OpenAIEmbeddings())
    result = vectorstore.similarity_search(query, k=1)
    return {"Result": result}

@app.get("/queryWithRetrieval")
async def query_with_retrieval(query : str = 'What are the approaches to Task Decomposition?', collection_name : str = 'temp1'):
    vectorstore = Chroma(collection_name=collection_name,persist_directory='./chroma_db',embedding_function = OpenAIEmbeddings())
    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
    qa_chain = RetrievalQAWithSourcesChain.from_chain_type(llm,retriever=vectorstore.as_retriever(search_kwargs={"k": 1}))
    result = qa_chain({"question": query})
    return {"result": result}

@app.get("/queryWithOutRetrieval")
async def query_without_retrieval(query : str = 'What are the approaches to Task Decomposition?', collection_name : str = 'temp1'):
    vectorstore = Chroma(collection_name=collection_name,persist_directory='./chroma_db',embedding_function = OpenAIEmbeddings())
    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
    result = llm.predict(query)
    return {"result": result}

paperspace-example/openai-langchain-basic-RAG/app.py
vultureprime/deploy-ai-model
import time
import torch
from transformers import (
    GenerationConfig,
    LlamaForCausalLM,
    LlamaTokenizer,
)
from peft import PeftModel
import transformers
import json
import os.path as osp
from typing import Union

base_model = "ChanonUtupon/openthaigpt-merge-lora-llama-2-7B-3470k"
lora_weights = 'openthaigpt/openthaigpt-1.0.0-beta-7b-chat'

class Prompter(object):
    __slots__ = ("template", "_verbose")

    def __init__(self, template_name: str = "", verbose: bool = False):
        self._verbose = verbose
        template_name = "alpaca"
        self.template = {
            "description": "Template used by Alpaca-LoRA.",
            "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
            "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n",
            "response_split": "### Response:"
        }
        if self._verbose:
            print(
                f"Using prompt template {template_name}: {self.template['description']}"
            )

    def generate_prompt(
        self,
        instruction: str,
        input: Union[None, str] = None,
        label: Union[None, str] = None,
    ) -> str:
        # returns the full prompt from instruction and optional input
        # if a label (=response, =output) is provided, it's also appended.
        if input:
            res = self.template["prompt_input"].format(
                instruction=instruction, input=input
            )
        else:
            res = self.template["prompt_no_input"].format(
                instruction=instruction
            )
        if label:
            res = f"{res}{label}"
        if self._verbose:
            print(res)
        return res

    def get_response(self, output: str) -> str:
        return output.split(self.template["response_split"])[1].strip()
prompt_template = ""
prompter = Prompter(prompt_template)
def run_eval(text_input): 
    init_start = time.time()
    
    prompt = prompter.generate_prompt(text_input, None)
    load_8bit = True
    tokenizer = LlamaTokenizer.from_pretrained(
        base_model,
        cache_dir="./llama2-openthaigpt",
        )
    model = LlamaForCausalLM.from_pretrained(
        base_model,
        load_in_8bit=load_8bit,
        cache_dir="./llama2-openthaigpt",
        torch_dtype=torch.float16,
        device_map="auto",
    )

    model = PeftModel.from_pretrained(
        model,
        lora_weights,
        cache_dir="./llama2-openthaigpt",
        torch_dtype=torch.float16,
        device_map="auto",
    )
    model.config.pad_token_id = tokenizer.pad_token_id = 0  # unk
    model.config.bos_token_id = 1
    model.config.eos_token_id = 2

    inputs = tokenizer(prompt, return_tensors="pt")
    input_ids = inputs["input_ids"].to("cuda")

    generation_config = GenerationConfig(
        # do_sample=True,
        temperature=0.1,
        top_p=0.75,
        top_k=40,
        num_beams=4,
    )
    start = time.time()
    with torch.no_grad():
        generation_output = model.generate(
            input_ids=input_ids,
            generation_config=generation_config,
            return_dict_in_generate=True,
            output_scores=True,
            max_new_tokens=128,
            early_stopping=True,
            repetition_penalty=1,
            no_repeat_ngram_size=0
        )

    s = generation_output.sequences[0]
    # decoded_output = tokenizer.decode(s, skip_special_tokens=True).strip()
    decoded_output = tokenizer.decode(s).split("### Response:")[1].strip()
    # print(tokenizer.decode(s))
    print(time.time() - start)
    print(time.time() - init_start)
    # print(generation_output.sequences)
    return {"pred": decoded_output}

print(run_eval('วิธีลดความอ้วนอย่างไร'))
paperspace-example/openthaigpt-1.0.0-beta-8bit-pytorch/app.py
vultureprime/deploy-ai-model
import time
import torch
from transformers import (
    GenerationConfig,
    LlamaForCausalLM,
    LlamaTokenizer,
)
from peft import PeftModel
import transformers
import json
import os.path as osp
from typing import Union

base_model = "ChanonUtupon/openthaigpt-merge-lora-llama-2-7B-3470k"
lora_weights = 'openthaigpt/openthaigpt-1.0.0-beta-7b-chat'

class Prompter(object):
    __slots__ = ("template", "_verbose")

    def __init__(self, template_name: str = "", verbose: bool = False):
        self._verbose = verbose
        template_name = "alpaca"
        self.template = {
            "description": "Template used by Alpaca-LoRA.",
            "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
            "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n",
            "response_split": "### Response:"
        }
        if self._verbose:
            print(
                f"Using prompt template {template_name}: {self.template['description']}"
            )

    def generate_prompt(
        self,
        instruction: str,
        input: Union[None, str] = None,
        label: Union[None, str] = None,
    ) -> str:
        # returns the full prompt from instruction and optional input
        # if a label (=response, =output) is provided, it's also appended.
        if input:
            res = self.template["prompt_input"].format(
                instruction=instruction, input=input
            )
        else:
            res = self.template["prompt_no_input"].format(
                instruction=instruction
            )
        if label:
            res = f"{res}{label}"
        if self._verbose:
            print(res)
        return res

    def get_response(self, output: str) -> str:
        return output.split(self.template["response_split"])[1].strip()
prompt_template = ""
prompter = Prompter(prompt_template)
def run_eval(text_input): 
    init_start = time.time()
    
    prompt = prompter.generate_prompt(text_input, None)
    load_8bit = False
    tokenizer = LlamaTokenizer.from_pretrained(
        base_model,
        cache_dir="./llama2-openthaigpt",
        )
    model = LlamaForCausalLM.from_pretrained(
        base_model,
        load_in_8bit=load_8bit,
        cache_dir="./llama2-openthaigpt",
        torch_dtype=torch.float16,
        device_map="auto",
    )
    model = PeftModel.from_pretrained(
        model,
        lora_weights,
        cache_dir="./llama2-openthaigpt",
        torch_dtype=torch.float16,
        device_map="auto",
    )
    model.config.pad_token_id = tokenizer.pad_token_id = 0  # unk
    model.config.bos_token_id = 1
    model.config.eos_token_id = 2

    inputs = tokenizer(prompt, return_tensors="pt")
    input_ids = inputs["input_ids"].to("cuda")

    generation_config = GenerationConfig(
        # do_sample=True,
        temperature=0.1,
        top_p=0.75,
        top_k=40,
        num_beams=4,
    )
    start = time.time()
    with torch.no_grad():
        generation_output = model.generate(
            input_ids=input_ids,
            generation_config=generation_config,
            return_dict_in_generate=True,
            output_scores=True,
            max_new_tokens=128,
            early_stopping=True,
            repetition_penalty=1,
            no_repeat_ngram_size=0
        )

    s = generation_output.sequences[0]
    # decoded_output = tokenizer.decode(s, skip_special_tokens=True).strip()
    decoded_output = tokenizer.decode(s).split("### Response:")[1].strip()
    # print(tokenizer.decode(s))
    print(time.time() - start)
    print(time.time() - init_start)
    # print(generation_output.sequences)
    return {"pred": decoded_output}

print(run_eval('วิธีลดความอ้วนอย่างไร'))
paperspace-example/openthaigpt-1.0.0-beta-pytorch/app.py
vultureprime/deploy-ai-model