Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tekken.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,583 @@
+---
+library_name: vllm
+language:
+- en
+- fr
+- es
+- de
+- it
+- pt
+- nl
+- zh
+- ja
+- ko
+- ar
+license: apache-2.0
+inference: false
+base_model:
+- mistralai/Ministral-3-14B-Reasoning-2512
+extra_gated_description: >-
+  If you want to learn more about how we process your personal data, please read
+  our <a href="https://mistral.ai/terms/">Privacy Policy</a>.
+tags:
+- mistral-common
+---
+
+# Ministral 3 14B Reasoning 2512
+
+The largest model in the Ministral 3 family, **Ministral 3 14B** offers frontier capabilities and performance comparable to its larger [Mistral Small 3.2 24B](https://huggingface.co/mistralai/Mistral-Small-3.2-Instruct-2506) counterpart. A powerful and efficient language model with vision capabilities.
+
+This model is the reasoning post-trained version, trained for reasoning tasks, making it ideal for math, coding and stem related use cases.
+
+The Ministral 3 family is designed for edge deployment, capable of running on a wide range of hardware. Ministral 3 14B can even be deployed locally, capable of fitting in 32GB of VRAM in BF16, and less than 24GB of RAM/VRAM when quantized.
+
+## Key Features
+Ministral 3 14B consists of two main architectural components:
+- **13.5B Language Model**
+- **0.4B Vision Encoder**
+
+The Ministral 3 14B Reasoning model offers the following capabilities:
+- **Vision**: Enables the model to analyze images and provide insights based on visual content, in addition to text.
+- **Multilingual**: Supports dozens of languages, including English, French, Spanish, German, Italian, Portuguese, Dutch, Chinese, Japanese, Korean, Arabic.
+- **System Prompt**: Maintains strong adherence and support for system prompts.
+- **Agentic**: Offers best-in-class agentic capabilities with native function calling and JSON outputting.
+- **Reasoning**: Excels at complex, multi-step reasoning and dynamic problem-solving.
+- **Edge-Optimized**: Delivers best-in-class performance at a small scale, deployable anywhere.
+- **Apache 2.0 License**: Open-source license allowing usage and modification for both commercial and non-commercial purposes.
+- **Large Context Window**: Supports a 256k context window.
+
+### Use Cases
+Private AI deployments where advanced capabilities meet practical hardware constraints:
+- Private/custom chat and AI assistant deployments in constrained environments
+- Advanced local agentic use cases
+- Fine-tuning and specialization
+- And more...
+  
+Bringing advanced AI capabilities to most environments.
+
+## Ministral 3 Family
+
+| Model Name                     | Type               | Precision | Link                                                                                     |
+|--------------------------------|--------------------|-----------|------------------------------------------------------------------------------------------|
+| Ministral 3 3B Base 2512       | Base pre-trained   | BF16      | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-3B-Base-2512)                |
+| Ministral 3 3B Instruct 2512   | Instruct post-trained | FP8   | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512)            |
+| Ministral 3 3B Reasoning 2512  | Reasoning capable  | BF16      | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512)           |
+| Ministral 3 8B Base 2512       | Base pre-trained   | BF16      | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-8B-Base-2512)                |
+| Ministral 3 8B Instruct 2512   | Instruct post-trained | FP8    | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512)            |
+| Ministral 3 8B Reasoning 2512  | Reasoning capable  | BF16      | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512)           |
+| Ministral 3 14B Base 2512      | Base pre-trained   | BF16      | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-14B-Base-2512)               |
+| Ministral 3 14B Instruct 2512  | Instruct post-trained | FP8    | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512)           |
+| **Ministral 3 14B Reasoning 2512** | **Reasoning capable**  | **BF16**      | [Hugging Face](https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512)          |
+
+Other formats available [here](https://huggingface.co/collections/mistralai/ministral-3-additional-checkpoints).
+
+## Benchmark Results
+
+We compare Ministral 3 to similar sized models.
+
+### Reasoning
+
+| Model                     | AIME25      | AIME24      | GPQA Diamond | LiveCodeBench |
+|---------------------------|-------------|-------------|--------------|---------------|
+| **Ministral 3 14B**       | <u>0.850</u>| <u>0.898</u>| <u>0.712</u> | <u>0.646</u>  |
+| Qwen3-14B (Thinking)      | 0.737       | 0.837       | 0.663        | 0.593         |
+|                           |             |             |              |               |
+| **Ministral 3 8B**        | 0.787       | <u>0.860</u>| 0.668        | <u>0.616</u>  |
+| Qwen3-VL-8B-Thinking      | <u>0.798</u>| <u>0.860</u>| <u>0.671</u> | 0.580         |
+|                           |             |             |              |               |
+| **Ministral 3 3B**        | <u>0.721</u>| <u>0.775</u>| 0.534        | <u>0.548</u>  |
+| Qwen3-VL-4B-Thinking      | 0.697       | 0.729       | <u>0.601</u> | 0.513         |
+
+### Instruct
+
+| Model                     | Arena Hard  | WildBench  | MATH Maj@1  | MM MTBench       |
+|---------------------------|-------------|------------|-------------|------------------|
+| **Ministral 3 14B**       | <u>0.551</u>| <u>68.5</u>| <u>0.904</u>| <u>8.49</u>      |
+| Qwen3 14B (Non-Thinking)  | 0.427       | 65.1       | 0.870       | NOT MULTIMODAL   |
+| Gemma3-12B-Instruct       | 0.436       | 63.2       | 0.854       | 6.70             |
+|                           |             |            |             |                  |
+| **Ministral 3 8B**        | 0.509       | <u>66.8</u>| 0.876       | <u>8.08</u>      |
+| Qwen3-VL-8B-Instruct      | <u>0.528</u>| 66.3       | <u>0.946</u>| 8.00             |
+|                           |             |            |             |                  |
+| **Ministral 3 3B**        | 0.305       | <u>56.8</u>| 0.830       | 7.83             |
+| Qwen3-VL-4B-Instruct      | <u>0.438</u>| <u>56.8</u>| <u>0.900</u>| <u>8.01</u>      |
+| Qwen3-VL-2B-Instruct      | 0.163       | 42.2       | 0.786       | 6.36             |
+| Gemma3-4B-Instruct        | 0.318       | 49.1       | 0.759       | 5.23             |
+
+### Base
+
+| Model               | Multilingual MMLU | MATH CoT 2-Shot | AGIEval 5-shot | MMLU Redux 5-shot | MMLU 5-shot | TriviaQA 5-shot |
+|---------------------|-------------------|-----------------|----------------|-------------------|-------------|-----------------|
+| **Ministral 3 14B** | 0.742             | <u>0.676</u>    | 0.648          | 0.820             | 0.794       | 0.749           |
+| Qwen3 14B Base      | <u>0.754</u>      | 0.620           | <u>0.661</u>   | <u>0.837</u>      | <u>0.804</u>| 0.703           |
+| Gemma 3 12B Base    | 0.690             | 0.487           | 0.587          | 0.766             | 0.745       | <u>0.788</u>    |
+|                     |                   |                 |                |                   |             |                 |
+| **Ministral 3 8B**  | <u>0.706</u>      | <u>0.626</u>    | 0.591          | 0.793             | <u>0.761</u>| <u>0.681</u>    |
+| Qwen 3 8B Base      | 0.700             | 0.576           | <u>0.596</u>   | <u>0.794</u>      | 0.760       | 0.639           |
+|                     |                   |                 |                |                   |             |                 |
+| **Ministral 3 3B**  | 0.652             | <u>0.601</u>    | 0.511          | 0.735             | 0.707       | 0.592           |
+| Qwen 3 4B Base      | <u>0.677</u>      | 0.405           | <u>0.570</u>   | <u>0.759</u>      | <u>0.713</u>| 0.530           |
+| Gemma 3 4B Base     | 0.516             | 0.294           | 0.430          | 0.626             | 0.589       | <u>0.640</u>    |
+
+## Usage
+
+The model can be used with the following frameworks;
+- [`vllm`](https://github.com/vllm-project/vllm): See [here](#vllm)
+- [`transformers`](https://github.com/huggingface/transformers): See [here](#transformers)
+  
+### vLLM
+
+We recommend using this model with [vLLM](https://github.com/vllm-project/vllm).
+
+#### Installation
+
+Make sure to install most recent vllm:
+
+```
+uv pip install -U vllm \
+    --torch-backend=auto \
+    --extra-index-url https://wheels.vllm.ai/nightly
+```
+
+Doing so should automatically install [`mistral_common >= 1.8.6`](https://github.com/mistralai/mistral-common/releases/tag/v1.8.6).
+
+To check:
+```
+python -c "import mistral_common; print(mistral_common.__version__)"
+```
+
+You can also make use of a ready-to-go [docker image](https://github.com/vllm-project/vllm/blob/main/Dockerfile) or on the [docker hub](https://hub.docker.com/layers/vllm/vllm-openai/latest/images/sha256-de9032a92ffea7b5c007dad80b38fd44aac11eddc31c435f8e52f3b7404bbf39).
+
+#### Serve
+
+To fully exploit the `Ministral-3-14B-Reasoning-2512` we recommed using 2xH200 GPUs for deployment due to its large context. However if you don't need a large context, you can fall back to a single GPU.
+
+A simple launch command is:
+
+```bash
+
+vllm serve mistralai/Ministral-3-14B-Reasoning-2512 \
+  --tensor-parallel-size 2 \
+  --tokenizer_mode mistral --config_format mistral --load_format mistral \
+  --enable-auto-tool-choice --tool-call-parser mistral \
+  --reasoning-parser mistral
+```
+
+Key parameter notes:
+
+* enable-auto-tool-choice: Required when enabling tool usage.
+* tool-call-parser mistral: Required when enabling tool usage.
+* reasoning-parser mistral: Required when enabling reasoning.
+
+Additional flags:
+
+* You can set `--max-model-len` to preserve memory. By default it is set to `262144` which is quite large but not necessary for most scenarios.
+* You can set `--max-num-batched-tokens` to balance throughput and latency, higher means higher throughput but higher latency.
+
+#### Usage of the model
+
+Here we asumme that the model `mistralai/Ministral-3-8B-Reasoning-2512` is served and you can ping it to the domain `localhost` with the port `8000` which is the default for vLLM.
+
+<details>
+  <summary>Vision Reasoning</summary>
+
+Let's see if the Ministral 3 model knows when to pick a fight !
+
+```python
+from typing import Any
+
+from openai import OpenAI
+from huggingface_hub import hf_hub_download
+
+# Modify OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:8000/v1"
+
+TEMP = 0.7
+TOP_P = 0.95
+MAX_TOK = 262144
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+models = client.models.list()
+model = models.data[0].id
+
+
+def load_system_prompt(repo_id: str, filename: str) -> dict[str, Any]:
+    file_path = hf_hub_download(repo_id=repo_id, filename=filename)
+    with open(file_path, "r") as file:
+        system_prompt = file.read()
+
+    index_begin_think = system_prompt.find("[THINK]")
+    index_end_think = system_prompt.find("[/THINK]")
+
+    return {
+        "role": "system",
+        "content": [
+            {"type": "text", "text": system_prompt[:index_begin_think]},
+            {
+                "type": "thinking",
+                "thinking": system_prompt[
+                    index_begin_think + len("[THINK]") : index_end_think
+                ],
+                "closed": True,
+            },
+            {
+                "type": "text",
+                "text": system_prompt[index_end_think + len("[/THINK]") :],
+            },
+        ],
+    }
+
+
+SYSTEM_PROMPT = load_system_prompt(model, "SYSTEM_PROMPT.txt")
+
+image_url = "https://static.wikia.nocookie.net/essentialsdocs/images/7/70/Battle.png/revision/latest?cb=20220523172438"
+
+messages = [
+    SYSTEM_PROMPT,
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "What action do you think I should take in this situation? List all the possible actions and explain why you think they are good or bad.",
+            },
+            {"type": "image_url", "image_url": {"url": image_url}},
+        ],
+    },
+]
+
+
+stream = client.chat.completions.create(
+    model=model,
+    messages=messages,
+    stream=True,
+    temperature=TEMP,
+    top_p=TOP_P,
+    max_tokens=MAX_TOK,
+)
+
+print("client: Start streaming chat completions...:\n")
+printed_reasoning_content = False
+answer = []
+
+for chunk in stream:
+    reasoning_content = None
+    content = None
+    # Check the content is reasoning_content or content
+    if hasattr(chunk.choices[0].delta, "reasoning_content"):
+        reasoning_content = chunk.choices[0].delta.reasoning_content
+    if hasattr(chunk.choices[0].delta, "content"):
+        content = chunk.choices[0].delta.content
+
+    if reasoning_content is not None:
+        if not printed_reasoning_content:
+            printed_reasoning_content = True
+            print("Start reasoning:\n", end="", flush=True)
+        print(reasoning_content, end="", flush=True)
+    elif content is not None:
+        # Extract and print the content
+        if not reasoning_content and printed_reasoning_content:
+            answer.extend(content)
+        print(content, end="", flush=True)
+
+if answer:
+    print("\n\n=============\nAnswer\n=============\n")
+    print("".join(answer))
+else:
+    print("\n\n=============\nNo Answer\n=============\n")
+    print(
+        "No answer was generated by the model, probably because the maximum number of tokens was reached."
+    )
+```
+
+Now we'll make it compute some maths !
+
+```python
+from typing import Any
+
+from openai import OpenAI
+from huggingface_hub import hf_hub_download
+
+# Modify OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:8000/v1"
+
+TEMP = 0.7
+TOP_P = 0.95
+MAX_TOK = 262144
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+models = client.models.list()
+model = models.data[0].id
+
+
+def load_system_prompt(repo_id: str, filename: str) -> dict[str, Any]:
+    file_path = hf_hub_download(repo_id=repo_id, filename=filename)
+    with open(file_path, "r") as file:
+        system_prompt = file.read()
+
+    index_begin_think = system_prompt.find("[THINK]")
+    index_end_think = system_prompt.find("[/THINK]")
+
+    return {
+        "role": "system",
+        "content": [
+            {"type": "text", "text": system_prompt[:index_begin_think]},
+            {
+                "type": "thinking",
+                "thinking": system_prompt[
+                    index_begin_think + len("[THINK]") : index_end_think
+                ],
+                "closed": True,
+            },
+            {
+                "type": "text",
+                "text": system_prompt[index_end_think + len("[/THINK]") :],
+            },
+        ],
+    }
+
+
+SYSTEM_PROMPT = load_system_prompt(model, "SYSTEM_PROMPT.txt")
+
+image_url = "https://i.ytimg.com/vi/5Y3xLHeyKZU/hqdefault.jpg"
+
+messages = [
+    SYSTEM_PROMPT,
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "Solve the equations. If they contain only numbers, use your calculator, else only think. Answer in the language of the image.",
+            },
+            {"type": "image_url", "image_url": {"url": image_url}},
+        ],
+    },
+]
+
+stream = client.chat.completions.create(
+    model=model,
+    messages=messages,
+    stream=True,
+    temperature=TEMP,
+    top_p=TOP_P,
+    max_tokens=MAX_TOK,
+)
+
+print("client: Start streaming chat completions...:\n")
+printed_reasoning_content = False
+answer = []
+
+for chunk in stream:
+    reasoning_content = None
+    content = None
+    # Check the content is reasoning_content or content
+    if hasattr(chunk.choices[0].delta, "reasoning_content"):
+        reasoning_content = chunk.choices[0].delta.reasoning_content
+    if hasattr(chunk.choices[0].delta, "content"):
+        content = chunk.choices[0].delta.content
+
+    if reasoning_content is not None:
+        if not printed_reasoning_content:
+            printed_reasoning_content = True
+            print("Start reasoning:\n", end="", flush=True)
+        print(reasoning_content, end="", flush=True)
+    if content is not None:
+        # Extract and print the content
+        if not reasoning_content and printed_reasoning_content:
+            answer.extend(content)
+        print(content, end="", flush=True)
+
+if answer:
+    print("\n\n=============\nAnswer\n=============\n")
+    print("".join(answer))
+else:
+    print("\n\n=============\nNo Answer\n=============\n")
+    print(
+        "No answer was generated by the model, probably because the maximum number of tokens was reached."
+    )
+```
+
+</details>
+
+<details>
+  <summary>Text-Only Request</summary>
+
+Let's do more maths and leave it up to the model to figure out how to achieve a result.
+
+```python
+from typing import Any
+from openai import OpenAI
+from huggingface_hub import hf_hub_download
+
+# Modify OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:8000/v1"
+
+TEMP = 0.7
+TOP_P = 0.95
+MAX_TOK = 262144
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+models = client.models.list()
+model = models.data[0].id
+
+
+def load_system_prompt(repo_id: str, filename: str) -> dict[str, Any]:
+    file_path = hf_hub_download(repo_id=repo_id, filename=filename)
+    with open(file_path, "r") as file:
+        system_prompt = file.read()
+
+    index_begin_think = system_prompt.find("[THINK]")
+    index_end_think = system_prompt.find("[/THINK]")
+
+    return {
+        "role": "system",
+        "content": [
+            {"type": "text", "text": system_prompt[:index_begin_think]},
+            {
+                "type": "thinking",
+                "thinking": system_prompt[
+                    index_begin_think + len("[THINK]") : index_end_think
+                ],
+                "closed": True,
+            },
+            {
+                "type": "text",
+                "text": system_prompt[index_end_think + len("[/THINK]") :],
+            },
+        ],
+    }
+
+
+SYSTEM_PROMPT = load_system_prompt(model, "SYSTEM_PROMPT.txt")
+
+query = "Use each number in 2,5,6,3 exactly once, along with any combination of +, -, ×, ÷ (and parentheses for grouping), to make the number 24."
+
+messages = [
+    SYSTEM_PROMPT,
+    {"role": "user", "content": query}
+]
+stream = client.chat.completions.create(
+  model=model,
+  messages=messages,
+  stream=True,
+  temperature=TEMP,
+  top_p=TOP_P,
+  max_tokens=MAX_TOK,
+)
+
+print("client: Start streaming chat completions...:\n")
+printed_reasoning_content = False
+answer = []
+
+for chunk in stream:
+    reasoning_content = None
+    content = None
+    # Check the content is reasoning_content or content
+    if hasattr(chunk.choices[0].delta, "reasoning_content"):
+        reasoning_content = chunk.choices[0].delta.reasoning_content
+    if hasattr(chunk.choices[0].delta, "content"):
+        content = chunk.choices[0].delta.content
+
+    if reasoning_content is not None:
+        if not printed_reasoning_content:
+            printed_reasoning_content = True
+            print("Start reasoning:\n", end="", flush=True)
+        print(reasoning_content, end="", flush=True)
+    if content is not None:
+        # Extract and print the content
+        if not reasoning_content and printed_reasoning_content:
+            answer.extend(content)
+        print(content, end="", flush=True)
+
+if answer:
+    print("\n\n=============\nAnswer\n=============\n")
+    print("".join(answer))
+else:
+    print("\n\n=============\nNo Answer\n=============\n")
+    print("No answer was generated by the model, probably because the maximum number of tokens was reached.")
+```
+
+</details>
+
+### Transformers
+
+You can also use Ministral 3 3B Reasoning 2512 with `Transformers` !
+Make sure to install `Transformers` from its first v5 release candidate or from "main":
+
+```
+pip install transformers==5.0.0rc0
+```
+
+To make the best use of our model with `Transformers` make sure to have [installed](https://github.com/mistralai/mistral-common) `mistral-common >= 1.8.6` to use our tokenizer.
+
+```bash
+pip install mistral-common --upgrade
+```
+
+Then load our tokenizer along with the model and generate:
+
+<details>
+  <summary>Python snippet</summary>
+
+```python
+import torch
+from transformers import Mistral3ForConditionalGeneration, MistralCommonBackend
+
+model_id = "mistralai/Ministral-3-14B-Reasoning-2512"
+
+tokenizer = MistralCommonBackend.from_pretrained(model_id)
+model = Mistral3ForConditionalGeneration.from_pretrained(
+    model_id, torch_dtype=torch.bfloat16, device_map="auto"
+)
+
+image_url = "https://static.wikia.nocookie.net/essentialsdocs/images/7/70/Battle.png/revision/latest?cb=20220523172438"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "What action do you think I should take in this situation? List all the possible actions and explain why you think they are good or bad.",
+            },
+            {"type": "image_url", "image_url": {"url": image_url}},
+        ],
+    },
+]
+
+tokenized = tokenizer.apply_chat_template(messages, return_tensors="pt", return_dict=True)
+
+tokenized["input_ids"] = tokenized["input_ids"].to(device="cuda")
+tokenized["pixel_values"] = tokenized["pixel_values"].to(dtype=torch.bfloat16, device="cuda")
+image_sizes = [tokenized["pixel_values"].shape[-2:]]
+
+output = model.generate(
+    **tokenized,
+    image_sizes=image_sizes,
+    max_new_tokens=8092,
+)[0]
+
+decoded_output = tokenizer.decode(output[len(tokenized["input_ids"][0]):])
+print(decoded_output)
+```
+
+</details>
+
+## License
+
+This model is licensed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.txt).
+
+*You must not use this model in a manner that infringes, misappropriates, or otherwise violates any third party’s rights, including intellectual property rights.*

SYSTEM_PROMPT.txt

@@ -0,0 +1,5 @@
+# HOW YOU SHOULD THINK AND ANSWER
+
+First draft your thinking process (inner monologue) until you arrive at a response. Format your response using Markdown, and use LaTeX for any mathematical equations. Write both your thoughts and the response in the same language as the input.
+
+Your thinking process must follow the template below:[THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate the response to the user.[/THINK]Here, provide a self-contained response.

chat_template.jinja

@@ -0,0 +1,126 @@
+{#- Default system message if no system prompt is passed. #}
+{%- set default_system_message = '# HOW YOU SHOULD THINK AND ANSWER\n\nFirst draft your thinking process (inner monologue) until you arrive at a response. Format your response using Markdown, and use LaTeX for any mathematical equations. Write both your thoughts and the response in the same language as the input.\n\nYour thinking process must follow the template below:[THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate the response to the user.[/THINK]Here, provide a self-contained response.' %}
+
+{#- Begin of sequence token. #}
+{{- bos_token }}
+
+{#- Handle system prompt if it exists. #}
+{#- System prompt supports text content or text and thinking chunks. #}
+{%- if messages[0]['role'] == 'system' %}
+    {{- '[SYSTEM_PROMPT]' -}}
+    {%- if messages[0]['content'] is string %}
+        {{- messages[0]['content'] -}}
+    {%- else %}        
+        {%- for block in messages[0]['content'] %}
+            {%- if block['type'] == 'text' %}
+                {{- block['text'] }}
+            {%- elif block['type'] == 'thinking' %}
+                {{- '[THINK]' + block['thinking'] + '[/THINK]' }}
+            {%- else %}
+                {{- raise_exception('Only text and thinking chunks are supported in system message contents.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- endif %}
+    {{- '[/SYSTEM_PROMPT]' -}}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set loop_messages = messages %}
+    {%- if default_system_message != '' %}
+        {{- '[SYSTEM_PROMPT]' + default_system_message + '[/SYSTEM_PROMPT]' }}
+    {%- endif %}
+{%- endif %}
+
+
+{#- Tools definition #}
+{%- set tools_definition = '' %}
+{%- set has_tools = false %}
+{%- if tools is defined and tools is not none and tools|length > 0 %}
+    {%- set has_tools = true %}
+    {%- set tools_definition = '[AVAILABLE_TOOLS]' + (tools| tojson) + '[/AVAILABLE_TOOLS]' %}
+    {{- tools_definition }}
+{%- endif %}
+
+{#- Checks for alternating user/assistant messages. #}
+{%- set ns = namespace(index=0) %}
+{%- for message in loop_messages %}
+    {%- if message.role == 'user' or (message.role == 'assistant' and (message.tool_calls is not defined or message.tool_calls is none or message.tool_calls | length == 0)) %}
+        {%- if (message['role'] == 'user') != (ns.index % 2 == 0) %}
+            {{- raise_exception('After the optional system message, conversation roles must alternate user and assistant roles except for tool calls and results.') }}
+        {%- endif %}
+        {%- set ns.index = ns.index + 1 %}
+    {%- endif %}
+{%- endfor %}
+
+{#- Handle conversation messages. #}
+{%- for message in loop_messages %}
+
+    {#- User messages supports text content or text and image chunks. #}
+    {%- if message['role'] == 'user' %}
+        {%- if message['content'] is string %}
+            {{- '[INST]' + message['content'] + '[/INST]' }}
+        {%- elif message['content'] | length > 0 %}
+            {{- '[INST]' }}
+            {%- if message['content'] | length == 2 %}
+                {%- set blocks = message['content'] | sort(attribute='type') %}
+            {%- else %}
+                {%- set blocks = message['content'] %}
+            {%- endif %}
+            {%- for block in blocks %}
+                {%- if block['type'] == 'text' %}
+                    {{- block['text'] }}
+                {%- elif block['type'] in ['image', 'image_url'] %}
+                    {{- '[IMG]' }}
+                {%- else %}
+                    {{- raise_exception('Only text, image and image_url chunks are supported in user message content.') }}
+                {%- endif %}
+            {%- endfor %}
+            {{- '[/INST]' }}
+        {%- else %}
+            {{- raise_exception('User message must have a string or a list of chunks in content') }}
+        {%- endif %}
+
+    {#- Assistant messages supports text content or text, image and thinking chunks. #}
+    {%- elif message['role'] == 'assistant' %}
+        {%- if (message['content'] is none or message['content'] == '' or message['content']|length == 0) and (message['tool_calls'] is not defined or message['tool_calls'] is none or message['tool_calls']|length == 0) %}
+            {{- raise_exception('Assistant message must have a string or a list of chunks in content or a list of tool calls.') }}
+        {%- endif %}
+
+        {%- if message['content'] is string and message['content'] != '' %}
+            {{- message['content'] }}
+        {%- elif message['content'] | length > 0 %}
+            {%- for block in message['content'] %}
+                {%- if block['type'] == 'text' %}
+                    {{- block['text'] }}
+                {%- elif block['type'] == 'thinking' %}
+                    {{- '[THINK]' + block['thinking'] + '[/THINK]' }}
+                {%- else %}
+                    {{- raise_exception('Only text and thinking chunks are supported in assistant message contents.') }}
+                {%- endif %}
+            {%- endfor %}
+        {%- endif %}
+        
+        {%- if message['tool_calls'] is defined and message['tool_calls'] is not none and message['tool_calls']|length > 0 %}
+            {%- for tool in message['tool_calls'] %}
+                {{- '[TOOL_CALLS]' }}
+                {%- set name = tool['function']['name'] %}
+                {%- set arguments = tool['function']['arguments'] %}
+                {%- if arguments is not string %}
+                    {%- set arguments = arguments|tojson|safe %}
+                {%- elif arguments == '' %}
+                    {%- set arguments = '{}' %}
+                {%- endif %}
+                {{- name + '[ARGS]' + arguments }}
+            {%- endfor %}
+        {%- endif %}
+
+        {{- eos_token }}
+
+    {#- Tool messages only supports text content. #}
+    {%- elif message['role'] == 'tool' %}
+        {{- '[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]' }}
+
+    {#- Raise exception for unsupported roles. #}
+    {%- else %}
+        {{- raise_exception('Only user, assistant and tool roles are supported, got ' + message['role'] + '.') }}
+    {%- endif %}
+{%- endfor %}

config.json

@@ -0,0 +1,266 @@
+{
+  "architectures": [
+    "Mistral3ForConditionalGeneration"
+  ],
+  "dtype": "bfloat16",
+  "image_token_index": 10,
+  "model_type": "mistral3",
+  "multimodal_projector_bias": false,
+  "projector_hidden_act": "gelu",
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 32,
+          "num_bits": 4,
+          "observer": "mse",
+          "observer_kwargs": {},
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int"
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.vision_tower.transformer.layers.0.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.0.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.0.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.0.attention.k_proj",
+      "model.vision_tower.transformer.layers.0.attention.v_proj",
+      "model.vision_tower.transformer.layers.0.attention.q_proj",
+      "model.vision_tower.transformer.layers.0.attention.o_proj",
+      "model.vision_tower.transformer.layers.1.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.1.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.1.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.1.attention.k_proj",
+      "model.vision_tower.transformer.layers.1.attention.v_proj",
+      "model.vision_tower.transformer.layers.1.attention.q_proj",
+      "model.vision_tower.transformer.layers.1.attention.o_proj",
+      "model.vision_tower.transformer.layers.2.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.2.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.2.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.2.attention.k_proj",
+      "model.vision_tower.transformer.layers.2.attention.v_proj",
+      "model.vision_tower.transformer.layers.2.attention.q_proj",
+      "model.vision_tower.transformer.layers.2.attention.o_proj",
+      "model.vision_tower.transformer.layers.3.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.3.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.3.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.3.attention.k_proj",
+      "model.vision_tower.transformer.layers.3.attention.v_proj",
+      "model.vision_tower.transformer.layers.3.attention.q_proj",
+      "model.vision_tower.transformer.layers.3.attention.o_proj",
+      "model.vision_tower.transformer.layers.4.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.4.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.4.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.4.attention.k_proj",
+      "model.vision_tower.transformer.layers.4.attention.v_proj",
+      "model.vision_tower.transformer.layers.4.attention.q_proj",
+      "model.vision_tower.transformer.layers.4.attention.o_proj",
+      "model.vision_tower.transformer.layers.5.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.5.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.5.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.5.attention.k_proj",
+      "model.vision_tower.transformer.layers.5.attention.v_proj",
+      "model.vision_tower.transformer.layers.5.attention.q_proj",
+      "model.vision_tower.transformer.layers.5.attention.o_proj",
+      "model.vision_tower.transformer.layers.6.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.6.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.6.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.6.attention.k_proj",
+      "model.vision_tower.transformer.layers.6.attention.v_proj",
+      "model.vision_tower.transformer.layers.6.attention.q_proj",
+      "model.vision_tower.transformer.layers.6.attention.o_proj",
+      "model.vision_tower.transformer.layers.7.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.7.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.7.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.7.attention.k_proj",
+      "model.vision_tower.transformer.layers.7.attention.v_proj",
+      "model.vision_tower.transformer.layers.7.attention.q_proj",
+      "model.vision_tower.transformer.layers.7.attention.o_proj",
+      "model.vision_tower.transformer.layers.8.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.8.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.8.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.8.attention.k_proj",
+      "model.vision_tower.transformer.layers.8.attention.v_proj",
+      "model.vision_tower.transformer.layers.8.attention.q_proj",
+      "model.vision_tower.transformer.layers.8.attention.o_proj",
+      "model.vision_tower.transformer.layers.9.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.9.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.9.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.9.attention.k_proj",
+      "model.vision_tower.transformer.layers.9.attention.v_proj",
+      "model.vision_tower.transformer.layers.9.attention.q_proj",
+      "model.vision_tower.transformer.layers.9.attention.o_proj",
+      "model.vision_tower.transformer.layers.10.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.10.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.10.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.10.attention.k_proj",
+      "model.vision_tower.transformer.layers.10.attention.v_proj",
+      "model.vision_tower.transformer.layers.10.attention.q_proj",
+      "model.vision_tower.transformer.layers.10.attention.o_proj",
+      "model.vision_tower.transformer.layers.11.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.11.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.11.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.11.attention.k_proj",
+      "model.vision_tower.transformer.layers.11.attention.v_proj",
+      "model.vision_tower.transformer.layers.11.attention.q_proj",
+      "model.vision_tower.transformer.layers.11.attention.o_proj",
+      "model.vision_tower.transformer.layers.12.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.12.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.12.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.12.attention.k_proj",
+      "model.vision_tower.transformer.layers.12.attention.v_proj",
+      "model.vision_tower.transformer.layers.12.attention.q_proj",
+      "model.vision_tower.transformer.layers.12.attention.o_proj",
+      "model.vision_tower.transformer.layers.13.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.13.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.13.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.13.attention.k_proj",
+      "model.vision_tower.transformer.layers.13.attention.v_proj",
+      "model.vision_tower.transformer.layers.13.attention.q_proj",
+      "model.vision_tower.transformer.layers.13.attention.o_proj",
+      "model.vision_tower.transformer.layers.14.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.14.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.14.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.14.attention.k_proj",
+      "model.vision_tower.transformer.layers.14.attention.v_proj",
+      "model.vision_tower.transformer.layers.14.attention.q_proj",
+      "model.vision_tower.transformer.layers.14.attention.o_proj",
+      "model.vision_tower.transformer.layers.15.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.15.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.15.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.15.attention.k_proj",
+      "model.vision_tower.transformer.layers.15.attention.v_proj",
+      "model.vision_tower.transformer.layers.15.attention.q_proj",
+      "model.vision_tower.transformer.layers.15.attention.o_proj",
+      "model.vision_tower.transformer.layers.16.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.16.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.16.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.16.attention.k_proj",
+      "model.vision_tower.transformer.layers.16.attention.v_proj",
+      "model.vision_tower.transformer.layers.16.attention.q_proj",
+      "model.vision_tower.transformer.layers.16.attention.o_proj",
+      "model.vision_tower.transformer.layers.17.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.17.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.17.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.17.attention.k_proj",
+      "model.vision_tower.transformer.layers.17.attention.v_proj",
+      "model.vision_tower.transformer.layers.17.attention.q_proj",
+      "model.vision_tower.transformer.layers.17.attention.o_proj",
+      "model.vision_tower.transformer.layers.18.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.18.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.18.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.18.attention.k_proj",
+      "model.vision_tower.transformer.layers.18.attention.v_proj",
+      "model.vision_tower.transformer.layers.18.attention.q_proj",
+      "model.vision_tower.transformer.layers.18.attention.o_proj",
+      "model.vision_tower.transformer.layers.19.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.19.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.19.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.19.attention.k_proj",
+      "model.vision_tower.transformer.layers.19.attention.v_proj",
+      "model.vision_tower.transformer.layers.19.attention.q_proj",
+      "model.vision_tower.transformer.layers.19.attention.o_proj",
+      "model.vision_tower.transformer.layers.20.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.20.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.20.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.20.attention.k_proj",
+      "model.vision_tower.transformer.layers.20.attention.v_proj",
+      "model.vision_tower.transformer.layers.20.attention.q_proj",
+      "model.vision_tower.transformer.layers.20.attention.o_proj",
+      "model.vision_tower.transformer.layers.21.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.21.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.21.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.21.attention.k_proj",
+      "model.vision_tower.transformer.layers.21.attention.v_proj",
+      "model.vision_tower.transformer.layers.21.attention.q_proj",
+      "model.vision_tower.transformer.layers.21.attention.o_proj",
+      "model.vision_tower.transformer.layers.22.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.22.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.22.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.22.attention.k_proj",
+      "model.vision_tower.transformer.layers.22.attention.v_proj",
+      "model.vision_tower.transformer.layers.22.attention.q_proj",
+      "model.vision_tower.transformer.layers.22.attention.o_proj",
+      "model.vision_tower.transformer.layers.23.feed_forward.gate_proj",
+      "model.vision_tower.transformer.layers.23.feed_forward.up_proj",
+      "model.vision_tower.transformer.layers.23.feed_forward.down_proj",
+      "model.vision_tower.transformer.layers.23.attention.k_proj",
+      "model.vision_tower.transformer.layers.23.attention.v_proj",
+      "model.vision_tower.transformer.layers.23.attention.q_proj",
+      "model.vision_tower.transformer.layers.23.attention.o_proj",
+      "model.multi_modal_projector.patch_merger.merging_layer",
+      "model.multi_modal_projector.linear_1",
+      "model.multi_modal_projector.linear_2",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.12.3.a20251114"
+  },
+  "spatial_merge_size": 2,
+  "text_config": {
+    "attention_dropout": 0.0,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 5120,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 262144,
+    "model_type": "ministral3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 40,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "beta_fast": 32.0,
+      "beta_slow": 1.0,
+      "factor": 16.0,
+      "llama_4_scaling_beta": 0.1,
+      "mscale": 1.0,
+      "mscale_all_dim": 1.0,
+      "original_max_position_embeddings": 16384,
+      "rope_theta": 1000000000.0,
+      "rope_type": "yarn",
+      "type": "yarn"
+    },
+    "sliding_window": null,
+    "use_cache": true,
+    "vocab_size": 131072
+  },
+  "transformers_version": "5.0.0.dev0",
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "image_size": 1540,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "model_type": "pixtral",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "vision_feature_layer": -1
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 262144,
+  "pad_token_id": 11,
+  "transformers_version": "5.0.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94595fb482260e76375b02c1c9e529dd93aef24f4e1b870f24a90f58d98df29d
+size 4997437320

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ba4c93fd826d1f46dc563dd2ab2e10137f21944472d19b7b4c0871e636e194c
+size 4996667240

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:367579ebb1886eedd0254816069243135c08b3ffc52cfd1ecba440d3b39f8550
+size 411107768

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7291c881406d3702616e00ac91512519534ce5500f6fdc698b03a3e4aab97a3
+size 10405191456

params.json

@@ -0,0 +1,47 @@
+{
+  "dim": 5120,
+  "n_layers": 40,
+  "head_dim": 128,
+  "hidden_dim": 16384,
+  "n_heads": 32,
+  "n_kv_heads": 8,
+  "rope_theta": 1000000000.0,
+  "norm_eps": 1e-05,
+  "vocab_size": 131072,
+  "tied_embeddings": false,
+  "max_position_embeddings": 262144,
+  "llama_4_scaling": {
+    "original_max_position_embeddings": 16384,
+    "beta": 0.1
+  },
+  "q_lora_rank": null,
+  "qk_rope_head_dim": null,
+  "qk_nope_head_dim": null,
+  "kv_lora_rank": null,
+  "v_head_dim": null,
+  "yarn": {
+    "original_max_position_embeddings": 16384,
+    "factor": 16,
+    "apply_scale": false,
+    "beta": 32,
+    "alpha": 1
+  },
+  "vision_encoder": {
+    "image_token_id": 10,
+    "intermediate_size": 4096,
+    "num_hidden_layers": 24,
+    "num_attention_heads": 16,
+    "mm_projector_id": "patch_merge",
+    "spatial_merge_size": 2,
+    "hidden_size": 1024,
+    "num_channels": 3,
+    "image_size": 1540,
+    "max_image_size": 1540,
+    "patch_size": 14,
+    "rope_theta": 10000.0,
+    "add_pre_mm_projector_layer_norm": true,
+    "adapter_bias": false,
+    "image_break_token_id": 12,
+    "image_end_token_id": 13
+  }
+}

processor_config.json

@@ -0,0 +1,42 @@
+{
+  "image_break_token": "[IMG_BREAK]",
+  "image_end_token": "[IMG_END]",
+  "image_processor": {
+    "crop_size": null,
+    "data_format": "channels_first",
+    "device": null,
+    "disable_grouping": null,
+    "do_center_crop": null,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_pad": null,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "PixtralImageProcessorFast",
+    "image_seq_length": null,
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "input_data_format": null,
+    "pad_size": null,
+    "patch_size": 14,
+    "processor_class": "PixtralProcessor",
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_tensors": null,
+    "size": {
+      "longest_edge": 1540
+    }
+  },
+  "image_token": "[IMG]",
+  "patch_size": 14,
+  "processor_class": "PixtralProcessor",
+  "spatial_merge_size": 2
+}

recipe.yaml

@@ -0,0 +1,37 @@
+default_stage:
+  default_modifiers:
+    AWQModifier:
+      config_groups:
+        group_0:
+          targets: [Linear]
+          weights:
+            num_bits: 4
+            type: int
+            symmetric: true
+            group_size: 32
+            strategy: group
+            block_structure: null
+            dynamic: false
+            actorder: null
+            scale_dtype: null
+            zp_dtype: null
+            observer: mse
+            observer_kwargs: {}
+          input_activations: null
+          output_activations: null
+          format: null
+      targets: [Linear]
+      ignore: ['re:.*embed_tokens', 're:.*input_layernorm', 're:.*post_attention_layernorm',
+        're:.*norm', 're:.*lm_head', 're:.*multi_modal_projector.*', 're:.*vision_tower.*']
+      mappings:
+      - smooth_layer: re:.*input_layernorm$
+        balance_layers: ['re:.*q_proj$', 're:.*k_proj$', 're:.*v_proj$']
+      - smooth_layer: re:.*v_proj$
+        balance_layers: ['re:.*o_proj$']
+      - smooth_layer: re:.*post_attention_layernorm$
+        balance_layers: ['re:.*gate_proj$', 're:.*up_proj$']
+      - smooth_layer: re:.*up_proj$
+        balance_layers: ['re:.*down_proj$']
+      offload_device: !!python/object/apply:torch.device [cpu]
+      duo_scaling: true
+      n_grid: 20

tekken.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e29d19ea32eb7e26e6c0572d57cb7f9eca0f4420e0e0fe6ae1cf3be94da1c0d6
+size 16753777

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:577575622324b2e099e2648be26bdeb5e5815ffe66d7004e9e3ddbf421db6bf1
+size 17078110