GPTQ model commit

config.json

@@ -0,0 +1,38 @@
+{
+    "_name_or_path": "/workspace/process/llmware_dragon-mistral-7b-v0/source",
+    "architectures": [
+        "MistralForCausalLM"
+    ],
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 14336,
+    "max_position_embeddings": 32768,
+    "model_type": "mistral",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 10000.0,
+    "sliding_window": 4096,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.35.2",
+    "use_cache": true,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "bits": 4,
+        "group_size": 128,
+        "damp_percent": 0.1,
+        "desc_act": true,
+        "sym": true,
+        "true_sequential": true,
+        "model_name_or_path": null,
+        "model_file_base_name": "model",
+        "quant_method": "gptq"
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.34.0.dev0"
+}

generation_test_hf_script.py

@@ -0,0 +1,89 @@
+
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+
+def load_rag_benchmark_tester_ds():
+
+    # pull 200 question rag benchmark test dataset from LLMWare HuggingFace repo
+    from datasets import load_dataset
+
+    ds_name = "llmware/rag_instruct_benchmark_tester"
+
+    dataset = load_dataset(ds_name)
+
+    print("update: loading RAG Benchmark test dataset - ", dataset)
+
+    test_set = []
+    for i, samples in enumerate(dataset["train"]):
+        test_set.append(samples)
+
+        # to view test set samples
+        # print("rag benchmark dataset test samples: ", i, samples)
+
+    return test_set
+
+
+def run_test(model_name, test_ds):
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    print("\nRAG Performance Test - 200 questions")
+    print("update: model - ", model_name)    
+    print("update: device - ", device)
+
+    model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+    model.to(device)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+    for i, entries in enumerate(test_ds):
+
+        # prepare prompt packaging used in fine-tuning process
+        new_prompt = "<human>: " + entries["context"] + "\n" + entries["query"] + "\n" + "<bot>:"
+
+        inputs = tokenizer(new_prompt, return_tensors="pt")
+        start_of_output = len(inputs.input_ids[0])
+
+        #   temperature: set at 0.3 for consistency of output
+        #   max_new_tokens:  set at 100 - may prematurely stop a few of the summaries
+
+        outputs = model.generate(
+            inputs.input_ids.to(device),
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.eos_token_id,
+            do_sample=True,
+            temperature=0.3,
+            max_new_tokens=100,
+            )
+
+        output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True)
+
+        # quick/optional post-processing clean-up of potential fine-tuning artifacts
+
+        eot = output_only.find("<|endoftext|>")
+        if eot > -1:
+            output_only = output_only[:eot]
+
+        bot = output_only.find("<bot>:")
+        if bot > -1:
+            output_only = output_only[bot+len("<bot>:"):]
+
+        # end - post-processing
+
+        print("\n")
+        print(i, "llm_response - ", output_only)
+        print(i, "gold_answer - ", entries["answer"])
+
+    return 0
+
+
+if __name__ == "__main__":
+
+    test_ds = load_rag_benchmark_tester_ds()
+
+    model_name = "llmware/dragon-mistral-7b-v0"
+    output = run_test(model_name,test_ds)
+
+

generation_test_llmware_script.py

@@ -0,0 +1,70 @@
+
+from llmware.prompts import Prompt
+
+
+def load_rag_benchmark_tester_ds():
+
+    # pull 200 question rag benchmark test dataset from LLMWare HuggingFace repo
+    from datasets import load_dataset
+
+    ds_name = "llmware/rag_instruct_benchmark_tester"
+
+    dataset = load_dataset(ds_name)
+
+    print("update: loading RAG Benchmark test dataset - ", dataset)
+
+    test_set = []
+    for i, samples in enumerate(dataset["train"]):
+        test_set.append(samples)
+
+        # to view test set samples
+        # print("rag benchmark dataset test samples: ", i, samples)
+
+    return test_set
+
+
+def run_test(model_name, prompt_list):
+
+    print("\nupdate: Starting RAG Benchmark Inference Test - ", model_name)
+
+    # pull DRAGON / BLING model directly from catalog, e.g., no from_hf=True
+    prompter = Prompt().load_model(model_name)
+
+    for i, entries in enumerate(prompt_list):
+
+        prompt = entries["query"]
+        context = entries["context"]
+
+        response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3)
+
+        print("\nupdate: model inference output - ", i, response["llm_response"])
+        print("update: gold_answer              - ", i, entries["answer"])
+
+        fc = prompter.evidence_check_numbers(response)
+        sc = prompter.evidence_comparison_stats(response)
+        sr = prompter.evidence_check_sources(response)
+
+        print("\nFact-Checking Tools")
+
+        for entries in fc:
+            for f, facts in enumerate(entries["fact_check"]):
+                print("update: fact check - ", f, facts)
+
+        for entries in sc:
+            print("update: comparison stats - ", entries["comparison_stats"])
+
+        for entries in sr:
+            for s, sources in enumerate(entries["source_review"]):
+                print("update: sources - ", s, sources)
+
+    return 0
+
+
+if __name__ == "__main__":
+
+    core_test_set = load_rag_benchmark_tester_ds()
+
+    # one of the 7 gpu dragon models
+    gpu_model_name = "llmware/dragon-mistral-7b-v0"
+
+    output = run_test(gpu_model_name, core_test_set)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f098226f0b8cc3e5b61a75261eb9afc7f6dbef6b9e3fb10ad8f6f1ccede1be54
+size 4158662280

quantize_config.json

@@ -0,0 +1,10 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "damp_percent": 0.1,
+  "desc_act": true,
+  "sym": true,
+  "true_sequential": true,
+  "model_name_or_path": null,
+  "model_file_base_name": "model"
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}