Upload FloSmolV

config.json

@@ -0,0 +1,383 @@
+{
+  "architectures": [
+    "FloSmolV"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_flosmolv.FloSmolVConfig",
+    "AutoModelForCausalLM": "modeling_flosmolv.FloSmolV"
+  },
+  "llm_config": {
+    "_name_or_path": "HuggingFaceTB/SmolLM-360M-Instruct",
+    "add_cross_attention": false,
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 960,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 2560,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 2048,
+    "min_length": 0,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 15,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 5,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 2,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 49152
+  },
+  "model_type": "flosmolV",
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0",
+  "vision_config": {
+    "_name_or_path": "microsoft/Florence-2-base",
+    "add_cross_attention": false,
+    "architectures": [
+      "Florence2ForConditionalGeneration"
+    ],
+    "auto_map": {
+      "AutoConfig": "microsoft/Florence-2-base--configuration_florence2.Florence2Config",
+      "AutoModelForCausalLM": "microsoft/Florence-2-base--modeling_florence2.Florence2ForConditionalGeneration"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "ignore_index": -100,
+    "is_decoder": false,
+    "is_encoder_decoder": true,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "florence2",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "prefix": null,
+    "problem_type": null,
+    "projection_dim": 768,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "text_config": {
+      "_name_or_path": "",
+      "activation_dropout": 0.1,
+      "activation_function": "gelu",
+      "add_bias_logits": false,
+      "add_cross_attention": false,
+      "add_final_layer_norm": false,
+      "architectures": null,
+      "attention_dropout": 0.1,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": 0,
+      "chunk_size_feed_forward": 0,
+      "classif_dropout": 0.1,
+      "classifier_dropout": 0.0,
+      "cross_attention_hidden_size": null,
+      "d_model": 768,
+      "decoder_attention_heads": 12,
+      "decoder_ffn_dim": 3072,
+      "decoder_layerdrop": 0.0,
+      "decoder_layers": 6,
+      "decoder_start_token_id": 2,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "dropout": 0.1,
+      "early_stopping": true,
+      "encoder_attention_heads": 12,
+      "encoder_ffn_dim": 3072,
+      "encoder_layerdrop": 0.0,
+      "encoder_layers": 6,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": 2,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": 0,
+      "forced_eos_token_id": 2,
+      "gradient_checkpointing": false,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1",
+        "2": "LABEL_2"
+      },
+      "init_std": 0.02,
+      "is_decoder": false,
+      "is_encoder_decoder": true,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1,
+        "LABEL_2": 2
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "max_position_embeddings": 1024,
+      "min_length": 0,
+      "model_type": "florence2_language",
+      "no_repeat_ngram_size": 3,
+      "normalize_before": false,
+      "num_beam_groups": 1,
+      "num_beams": 3,
+      "num_hidden_layers": 6,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": 1,
+      "prefix": null,
+      "problem_type": null,
+      "pruned_heads": {},
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "scale_embedding": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "typical_p": 1.0,
+      "use_bfloat16": false,
+      "use_cache": true,
+      "vocab_size": 51289
+    },
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vision_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "depths": [
+        1,
+        1,
+        9,
+        1
+      ],
+      "dim_embed": [
+        128,
+        256,
+        512,
+        1024
+      ],
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "drop_path_rate": 0.1,
+      "early_stopping": false,
+      "enable_checkpoint": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "image_feature_source": [
+        "spatial_avg_pool",
+        "temporal_avg_pool"
+      ],
+      "image_pos_embed": {
+        "max_pos_embeddings": 50,
+        "type": "learned_abs_2d"
+      },
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_groups": [
+        4,
+        8,
+        16,
+        32
+      ],
+      "num_heads": [
+        4,
+        8,
+        16,
+        32
+      ],
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "patch_padding": [
+        3,
+        1,
+        1,
+        1
+      ],
+      "patch_prenorm": [
+        false,
+        true,
+        true,
+        true
+      ],
+      "patch_size": [
+        7,
+        3,
+        3,
+        3
+      ],
+      "patch_stride": [
+        4,
+        2,
+        2,
+        2
+      ],
+      "prefix": null,
+      "problem_type": null,
+      "projection_dim": 768,
+      "pruned_heads": {},
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "typical_p": 1.0,
+      "use_bfloat16": false,
+      "visual_temporal_embedding": {
+        "max_temporal_embeddings": 100,
+        "type": "COSINE"
+      },
+      "window_size": 12
+    },
+    "vocab_size": 51289
+  }
+}

configuration_flosmolv.py

@@ -0,0 +1,20 @@
+from transformers import AutoConfig, PretrainedConfig
+
+class FloSmolVConfig(PretrainedConfig):
+    model_type = "flosmolV"
+    is_composition = True 
+
+    def __init__(
+        self, 
+        vision_config=None,
+        llm_config=None,
+        **kwargs,
+      ):
+
+        if vision_config is None:
+            vision_config = AutoConfig.from_pretrained("microsoft/Florence-2-base", trust_remote_code=True)
+        if llm_config is None:
+            llm_config = AutoConfig.from_pretrained("HuggingFaceTB/SmolLM-360M-Instruct", trust_remote_code=True)
+        self.vision_config = vision_config
+        self.llm_config = llm_config
+        super().__init__(**kwargs)

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f5dd2bcf0af7477ad0cff0de5270263564167ded8c7e94ccc71eb07489ce3c1
-size 2531249132
+oid sha256:4655e21a7276883f9db1dec7880a0c192c37b6b5e2db6cec9c25e72c8c0e7052
+size 2531248820

modeling_flosmolv.py

@@ -0,0 +1,55 @@
+from transformers import PreTrainedModel
+from typing import List
+import torch 
+from .configuration_flosmolv import FloSmolVConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor
+
+class FloSmolV(PreTrainedModel):
+    config_class = FloSmolVConfig
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Device: {device}")
+    def __init__(self, config):
+        super().__init__(config)
+        self.florence2_model = AutoModelForCausalLM.from_pretrained(
+            self.config.vision_config._name_or_path,
+            trust_remote_code=True,
+        ).eval().to(self.device)
+        self.florence2_processor = AutoProcessor.from_pretrained(self.config.vision_config._name_or_path, trust_remote_code=True,)
+        self.smollm_model = AutoModelForCausalLM.from_pretrained(
+            self.config.llm_config._name_or_path,
+            trust_remote_code=True,
+        ).to(self.device)
+        self.smollm_tokenizer = AutoTokenizer.from_pretrained(self.config.llm_config._name_or_path, trust_remote_code=True,)
+      
+    def forward(self, image, query: str):
+      ## Vision
+        prompt = "<MORE_DETAILED_CAPTION>"
+        vision_inpupt = self.florence2_processor(text=prompt, images=image, return_tensors="pt")
+        generated_ids = self.florence2_model.generate(
+            input_ids=vision_inpupt["input_ids"].to(torch.int64).to(self.device),
+            pixel_values=vision_inpupt["pixel_values"].to(self.device),
+            max_new_tokens=1024,
+            early_stopping=False,
+            do_sample=False,
+            num_beams=3,
+        )
+        generated_text = self.florence2_processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+        parsed_answer = self.florence2_processor.post_process_generation(
+            generated_text,
+            task=prompt,
+            image_size=(image.width, image.height)
+        )
+        ## LM
+        messages = [{"role": "user", "content": f"You are an expert AI assistant. Based on the CONTENT, you should answer the QUESTION in short.\n\nCONTENT:{str(parsed_answer[prompt])}\n\nQUESTION:{str(query)}\n"}]
+        input_text=self.smollm_tokenizer.apply_chat_template(messages, tokenize=False)
+        llm_inputs = self.smollm_tokenizer.encode(input_text, return_tensors="pt").to(self.device)
+        outputs = self.smollm_model.generate(
+            llm_inputs, 
+            max_new_tokens=50, 
+            temperature=0.2, 
+            # top_p=0.9, 
+            do_sample=True,
+        )
+        response = self.smollm_tokenizer.decode(outputs[0])
+        cleaned_text = response.split("assistant\n", 1)[-1].strip()
+        return cleaned_text[:-10] if cleaned_text[-10:] == "<|im_end|>" else cleaned_text