[2025-10-30 00:38:06,468] [DEBUG] [axolotl.utils.config.resolve_dtype:66] [PID:3449] bf16 support detected, enabling for this configuration.

config.json:   0%|                                                                                                                                                                                                                    | 0.00/601 [00:00<?, ?B/s]
config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 601/601 [00:00<00:00, 3.43MB/s]
[2025-10-30 00:38:06,970] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:3449] baseline 0.000GB ()
[2025-10-30 00:38:06,971] [INFO] [axolotl.cli.config.load_cfg:248] [PID:3449] config:
{
  "activation_offloading": false,
  "adapter": "lora",
  "axolotl_config_path": "config.yml",
  "base_model": "mistralai/Mistral-7B-Instruct-v0.3",
  "base_model_config": "mistralai/Mistral-7B-Instruct-v0.3",
  "batch_size": 16,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_86",
    "fp8": false,
    "n_gpu": 1,
    "n_node": 1
  },
  "chat_template": "mistral_v1",
  "context_parallel_size": 1,
  "dataloader_num_workers": 1,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 9,
  "datasets": [
    {
      "chat_template": "tokenizer_default",
      "field_messages": "messages",
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "VinceGx33/mistral-legal-french-dataset",
      "split": "train",
      "trust_remote_code": false,
      "type": "chat_template"
    }
  ],
  "ddp": false,
  "default_system_message": "",
  "device": "cuda:0",
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "env_capabilities": {
    "torch_version": "2.7.1"
  },
  "eval_batch_size": 4,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 512,
  "eval_steps": 500,
  "eval_table_size": 5,
  "experimental_skip_move_to_device": true,
  "flash_attention": false,
  "fp16": false,
  "gradient_accumulation_steps": 4,
  "gradient_checkpointing": true,
  "gradient_checkpointing_kwargs": {
    "use_reentrant": false
  },
  "hub_model_id": "VinceGx33/mistral-7b-legal-french-v0.3",
  "include_tkps": true,
  "is_falcon_derived_model": false,
  "is_llama_derived_model": false,
  "is_mistral_derived_model": true,
  "learning_rate": 0.0002,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 10,
  "lora_alpha": 32,
  "lora_dropout": 0.05,
  "lora_r": 16,
  "lora_target_linear": true,
  "lora_target_modules": [
    "q_proj",
    "v_proj",
    "k_proj",
    "o_proj",
    "gate_proj",
    "up_proj",
    "down_proj"
  ],
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "max_grad_norm": 1.0,
  "mean_resizing_embeddings": false,
  "micro_batch_size": 4,
  "model_config_type": "mistral",
  "num_epochs": 3.0,
  "optimizer": "adamw_8bit",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "/workspace/mistral-legal-finetuned",
  "pad_to_sequence_len": true,
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "sample_packing": false,
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_steps": 500,
  "save_total_limit": 3,
  "sdp_attention": false,
  "sequence_len": 2048,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": false,
  "skip_prepare_dataset": false,
  "special_tokens": {
    "pad_token": "<pad>"
  },
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": true,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "mistralai/Mistral-7B-Instruct-v0.3",
  "tokenizer_save_jinja_files": true,
  "tokenizer_type": "LlamaTokenizer",
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "scale_rewards": true,
    "sync_ref_model": false,
    "use_vllm": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "trust_remote_code": true,
  "type_of_model": "MistralForCausalLM",
  "use_otel_metrics": false,
  "use_ray": false,
  "use_wandb": true,
  "val_set_size": 0.05,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "wandb_log_model": "",
  "wandb_name": "mistral-7b-legal-v0.3",
  "wandb_project": "mistral-legal-french",
  "wandb_watch": "",
  "warmup_steps": 100,
  "weight_decay": 0.01,
  "world_size": 1,
  "xformers_attention": true
}

tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 141kB [00:00, 40.5MB/s]

tokenizer.model:   0%|                                                                                                                                                                                                               | 0.00/587k [00:00<?, ?B/s]
tokenizer.model: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 587k/587k [00:00<00:00, 819kB/s]
tokenizer.model: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 587k/587k [00:00<00:00, 818kB/s]

special_tokens_map.json:   0%|                                                                                                                                                                                                        | 0.00/414 [00:00<?, ?B/s]
special_tokens_map.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 414/414 [00:00<00:00, 1.46MB/s]

tokenizer.json: 0.00B [00:00, ?B/s]
tokenizer.json: 1.79MB [00:00, 12.5MB/s]
tokenizer.json: 1.96MB [00:00, 13.1MB/s]
[2025-10-30 00:38:10,859] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:3449] EOS: 2 / </s>
[2025-10-30 00:38:10,859] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:3449] BOS: 1 / <s>
[2025-10-30 00:38:10,859] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:3449] PAD: 32768 / <pad>
[2025-10-30 00:38:10,860] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:3449] UNK: 0 / <unk>
[2025-10-30 00:38:10,860] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:3449] Unable to find prepared dataset in last_run_prepared/7b29567c6088f93eb77efbc6c1f13463
[2025-10-30 00:38:10,860] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:3449] Loading raw datasets...
[2025-10-30 00:38:10,860] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:3449] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.

README.md: 0.00B [00:00, ?B/s]
README.md: 21.6kB [00:00, 24.9MB/s]

dataset_autotrain_curriculum.jsonl:   0%|                                                                                                                                                                                           | 0.00/21.7M [00:00<?, ?B/s]
dataset_autotrain_curriculum.jsonl: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 21.7M/21.7M [00:01<00:00, 13.9MB/s]
dataset_autotrain_curriculum.jsonl: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 21.7M/21.7M [00:01<00:00, 13.8MB/s]

Generating train split:   0%|                                                                                                                                                                                                  | 0/14875 [00:00<?, ? examples/s]
Generating train split:  70%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                     | 10420/14875 [00:00<00:00, 86374.19 examples/s]
Generating train split: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 14875/14875 [00:00<00:00, 58112.66 examples/s]
[2025-10-30 00:38:15,488] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:3449] Loading dataset: VinceGx33/mistral-legal-french-dataset with base_type: chat_template and prompt_style: None
[2025-10-30 00:38:15,518] [INFO] [axolotl.prompt_strategies.chat_template.__call__:969] [PID:3449] Using chat template:
---
{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ ' [INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}

---

Tokenizing Prompts (num_proc=9):   0%|                                                                                                                                                                                         | 0/14875 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=9):   0%|                                                                                                                                                                                         | 0/14875 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=9):   0%|                                                                                                                                                                                         | 0/14875 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=9):   7%|███████████▋                                                                                                                                                                 | 1000/14875 [00:03<00:31, 436.88 examples/s]
Tokenizing Prompts (num_proc=9):  13%|███████████████████████▎                                                                                                                                                     | 2000/14875 [00:03<00:14, 899.73 examples/s]
Tokenizing Prompts (num_proc=9):  27%|██████████████████████████████████████████████▎                                                                                                                             | 4000/14875 [00:03<00:04, 2196.63 examples/s]
Tokenizing Prompts (num_proc=9):  40%|█████████████████████████████████████████████████████████████████████▍                                                                                                      | 6000/14875 [00:03<00:02, 3770.05 examples/s]
Tokenizing Prompts (num_proc=9):  49%|████████████████████████████████████████████████████████████████████████████████████▍                                                                                       | 7306/14875 [00:05<00:04, 1753.12 examples/s]
Tokenizing Prompts (num_proc=9):  58%|███████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                        | 8612/14875 [00:05<00:02, 2374.64 examples/s]
Tokenizing Prompts (num_proc=9):  67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                         | 9918/14875 [00:05<00:01, 2924.22 examples/s]
Tokenizing Prompts (num_proc=9):  67%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                         | 9918/14875 [00:05<00:02, 1740.84 examples/s]
multiprocess.pool.RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/multiprocess/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 586, in _write_generator_to_queue
    for i, result in enumerate(func(**kwargs)):
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3674, in _map_single
    for i, batch in iter_outputs(shard_iterable):
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3624, in iter_outputs
    yield i, apply_function(example, i, offset=offset)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3547, in apply_function
    processed_inputs = function(*fn_args, *additional_args, **fn_kwargs)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/prompt_strategies/chat_template.py", line 405, in tokenize_prompt
    tokenized_prompt = self._tokenize_single_prompt(
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/prompt_strategies/chat_template.py", line 458, in _tokenize_single_prompt
    input_ids = self.prompter.build_prompt(turns, tools=tools)  # type: ignore
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/prompt_strategies/chat_template.py", line 147, in build_prompt
    return self.tokenizer.apply_chat_template(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/tokenization_utils_base.py", line 1666, in apply_chat_template
    rendered_chat, generation_indices = render_jinja_template(
                                        ^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/chat_template_utils.py", line 539, in render_jinja_template
    rendered_chat = compiled_template.render(
                    ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/jinja2/environment.py", line 1295, in render
    self.environment.handle_exception()
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/jinja2/environment.py", line 942, in handle_exception
    raise rewrite_traceback_stack(source=source)
  File "<template>", line 1, in top-level template code
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/jinja2/sandbox.py", line 401, in call
    return __context.call(__obj, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/chat_template_utils.py", line 447, in raise_exception
    raise jinja2.exceptions.TemplateError(message)
jinja2.exceptions.TemplateError: Conversation roles must alternate user/assistant/user/assistant/...
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
    fire.Fire(do_cli)
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
    component_trace = _Fire(component, args, parsed_flag_args, context, name)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
    component, remaining_args = _CallAndUpdateTrace(
                                ^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
    component = fn(*varargs, **kwargs)
                ^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
    return do_train(parsed_cfg, parsed_cli_args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 43, in do_train
    dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/common/datasets.py", line 59, in load_datasets
    train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets(
                                                              ^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets
    return _prepare_standard_dataset(cfg, tokenizer, processor)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset
    train_dataset, eval_dataset, prompters = loader.load(_load_datasets)
                                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/lock.py", line 38, in load
    result = load_fn()
             ^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets
    train_dataset, eval_dataset, prompters = _load_and_prepare_datasets(
                                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 503, in _load_and_prepare_datasets
    dataset, prompters = _load_tokenized_prepared_datasets(
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets
    dataset, prompters = _load_raw_datasets(
                         ^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets
    dataset_wrapper, dataset_prompter = _load_and_process_single_dataset(
                                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 401, in _load_and_process_single_dataset
    dataset_wrapper, dataset_prompter = get_dataset_wrapper(
                                        ^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/wrappers.py", line 123, in get_dataset_wrapper
    return _handle_loaded_strategy(dataset_strategy, dataset, dataset_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/utils/data/wrappers.py", line 223, in _handle_loaded_strategy
    dataset_wrapper = wrap_dataset_for_tokenized_prompt(
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/datasets.py", line 87, in wrap_dataset_for_tokenized_prompt
    return TokenizedPromptDataset(prompt_tokenizer, dataset, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/datasets.py", line 40, in __init__
    self.process(dataset).data,
    ^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/datasets.py", line 62, in process
    return dataset.map(
           ^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 560, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
                                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3309, in map
    for rank, done, content in iflatmap_unordered(
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 626, in iflatmap_unordered
    [async_result.get(timeout=0.05) for async_result in async_results]
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 626, in <listcomp>
    [async_result.get(timeout=0.05) for async_result in async_results]
     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/multiprocess/pool.py", line 774, in get
    raise self._value
jinja2.exceptions.TemplateError: Conversation roles must alternate user/assistant/user/assistant/...