

- [Access Colab](https://colab.research.google.com/drive/1k0qae5AEd_Zdj3rYDS6Nl_dWl97szVNs?usp=sharing)

# how to use the model.
```python
# First install the following library
#!pip install -q "transformers" "datasets" "peft" "accelerate" "bitsandbytes" "safetensors"

import torch
from peft import AutoPeftModelForCausalLM
from transformers import AutoTokenizer

path= "Imran1/starling_base_sft"

# load base LLM model and tokenizer
model = AutoPeftModelForCausalLM.from_pretrained(
    path,
    low_cpu_mem_usage=True,
    torch_dtype=torch.float16,
    load_in_4bit=True,
)
tokenizer = AutoTokenizer.from_pretrained(path)


tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"



texts = '''{'role': 'user', 'content': '<s> calculate the student % and CGPA and give me example using python with output.'},
{'role': 'assistant', 'content': '''

inputs = tokenizer(texts, return_tensors="pt").to("cuda")
generation_params = {
    'max_length': 500,
    'use_cache': True,
    'do_sample': True,      
    'temperature': 0.7,    
    'top_p': 0.9,          
    'top_k': 50             
}

outputs = model.generate(**inputs, **generation_params)
decoded_outputs = tokenizer.batch_decode(outputs)

for text in decoded_outputs:
    text = text.replace('\\n', '\n')
    print(text)
    print("\n\n")

{}
safetensors region:us