import gradio as gr import spaces import torch from torch.cuda.amp import autocast import subprocess from huggingface_hub import InferenceClient import os import psutil """ For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference """ from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch from accelerate import Accelerator subprocess.run( "pip install psutil", shell=True, ) import bitsandbytes as bnb # Import bitsandbytes for 8-bit quantization from datetime import datetime subprocess.run( "pip install flash-attn --no-build-isolation", env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"}, shell=True, ) client = InferenceClient("HuggingFaceH4/zephyr-7b-beta") # pip install 'git+https://github.com/huggingface/transformers.git' token=os.getenv('token') print('token = ',token) from transformers import AutoModelForCausalLM, AutoTokenizer # model_id = "mistralai/Mistral-7B-v0.3" # model_id = "openchat/openchat-3.6-8b-20240522" model_id = "meta-llama/Meta-Llama-3-8B-Instruct" tokenizer = AutoTokenizer.from_pretrained( # model_id model_id , token= token,) accelerator = Accelerator() model = AutoModelForCausalLM.from_pretrained(model_id, token= token, # torch_dtype= torch.uint8, torch_dtype=torch.bfloat16, # load_in_4bit=True, # # # torch_dtype=torch.fl, attn_implementation="flash_attention_2", low_cpu_mem_usage=True, # device_map='cuda', # device_map=accelerator.device_map, ) # model = accelerator.prepare(model) ################################################### BG REMOVER ################################################### import gradio as gr from gradio_imageslider import ImageSlider from loadimg import load_img import spaces from transformers import AutoModelForImageSegmentation import torch from torchvision import transforms torch.set_float32_matmul_precision(["high", "highest"][0]) birefnet = AutoModelForImageSegmentation.from_pretrained( "ZhengPeng7/BiRefNet", trust_remote_code=True ) birefnet.to("cuda") transform_image = transforms.Compose( [ transforms.Resize((1024, 1024)), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), ] ) import base64 from io import BytesIO from PIL import Image def convert_image_to_base64(image): """ Convert a PIL Image with alpha channel to a base64-encoded string. """ # Save the image into a BytesIO buffer img_byte_array = BytesIO() image.save(img_byte_array, format="PNG") # Use PNG for transparency img_byte_array.seek(0) # Reset the pointer to the beginning # Encode the image bytes to base64 base64_str = base64.b64encode(img_byte_array.getvalue()).decode("utf-8") return base64_str import json def str_to_json(str_obj): json_obj = json.loads(str_obj) return json_obj @spaces.GPU(duration=140) def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, ): # yield 'retuend' # model.to(accelerator.device) messages = [] json_obj = str_to_json(message) print(json_obj) messages= json_obj try: image= json_obj['image'] print('selected bg remover') image = load_img(image, output_type="pil") image = image.convert("RGB") image_size = image.size input_images = transform_image(image).unsqueeze(0).to("cuda") # Prediction with torch.no_grad(): preds = birefnet(input_images)[-1].sigmoid().cpu() pred = preds[0].squeeze() pred_pil = transforms.ToPILImage()(pred) mask = pred_pil.resize(image_size) image.putalpha(mask) print('remver success') try: yield str(convert_image_to_base64(image)) except Exception as e: print(e) yield image except Exception as e: print("using llama 8b intrcuxt ",e) input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(accelerator.device) input_ids2 = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt") #.to('cuda') print(f"Converted input_ids dtype: {input_ids.dtype}") input_str= str(input_ids2) print('input str = ', input_str) with torch.no_grad(): gen_tokens = model.generate( input_ids, max_new_tokens=max_tokens, # do_sample=True, temperature=temperature, ) gen_text = tokenizer.decode(gen_tokens[0]) print(gen_text) gen_text= gen_text.replace(input_str,'') gen_text= gen_text.replace('<|eot_id|>','') yield gen_text # messages = [ # # {"role": "user", "content": "What is your favourite condiment?"}, # # {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"}, # # {"role": "user", "content": "Do you have mayonnaise recipes?"} # ] # inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to("cuda") # outputs = model.generate(inputs, max_new_tokens=2000) # gen_text=tokenizer.decode(outputs[0], skip_special_tokens=True) # print(gen_text) # yield gen_text # for val in history: # if val[0]: # messages.append({"role": "user", "content": val[0]}) # if val[1]: # messages.append({"role": "assistant", "content": val[1]}) # messages.append({"role": "user", "content": message}) # response = "" # for message in client.chat_completion( # messages, # max_tokens=max_tokens, # stream=True, # temperature=temperature, # top_p=top_p, # ): # token = message.choices[0].delta.content # response += token # yield response """ For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface """ demo = gr.ChatInterface( respond, additional_inputs=[ gr.Textbox(value="You are a friendly Chatbot.", label="System message"), gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)", ), ], ) if __name__ == "__main__": demo.launch()