Update README.md

README.md

@@ -129,7 +129,7 @@ base_model:
 
 ## Model Summary
 
-The LLaVA-NeXT-Video models are 7/72B parameter models trained on [LLaVA-NeXT-Video-178K](https://huggingface.co/datasets/lmms-lab/LLaVA-NeXT-Video-SFT-Data), based on Qwen2 language model with a context window of 32K tokens.
+In contrast to lmms-lab/LLaVA-NeXT-Video-7B-Qwen2, this is a 7B model trained on [LLaVA-NeXT-Video-178K](https://huggingface.co/datasets/lmms-lab/LLaVA-NeXT-Video-SFT-Data) only, based on Qwen2 language model with a context window of 32K tokens.
 
 This model support at most 110 frames.
 
@@ -183,19 +183,20 @@ def load_video(self, video_path, max_frames_num,fps=1,force_sample=False):
     spare_frames = vr.get_batch(frame_idx).asnumpy()
     # import pdb;pdb.set_trace()
     return spare_frames,frame_time,video_time
-pretrained = "lmms-lab/LLaVA-NeXT-Video-7B-Qwen2-Video-Only"
+pretrained = "lmms-lab/LLaVA-NeXT-Video-7B-Qwen2-Video-Only "
 model_name = "llava_qwen"
 device = "cuda"
 device_map = "auto"
-tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map)  # Add any other thing you want to pass in llava_model_args
+tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, torch_dtype="bfloat16", device_map=device_map)  # Add any other thing you want to pass in llava_model_args
 model.eval()
 video_path = "XXXX"
-max_frames_num = "110"
+max_frames_num = "64"
 video,frame_time,video_time = load_video(video_path, max_frames_num, 1, force_sample=True)
 video = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].cuda().bfloat16()
 video = [video]
 conv_template = "qwen_1_5"  # Make sure you use correct chat template for different models
-question = DEFAULT_IMAGE_TOKEN + "\nPlease describe this video in detail."
+time_instruciton = f"The video lasts for {video_time:.2f} seconds, and {len(video[0])} frames are uniformly sampled from it. These frames are located at {frame_time}.Please answer the following questions related to this video."
+question = DEFAULT_IMAGE_TOKEN + f"{time_instruciton}\nPlease describe this video in detail."
 conv = copy.deepcopy(conv_templates[conv_template])
 conv.append_message(conv.roles[0], question)
 conv.append_message(conv.roles[1], None)
@@ -204,12 +205,12 @@ input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX,
 cont = model.generate(
     input_ids,
     images=video,
-    modalities=["video"],
+    modalities= ["video"],
     do_sample=False,
     temperature=0,
     max_new_tokens=4096,
 )
-text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)
+text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)[0].strip()
 print(text_outputs)
 ```