Training in progress, epoch 0

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+20250401_073519_Jessie_gemma-3-27b-it_e0_merged.png filter=lfs diff=lfs merge=lfs -text
+20250401_073951_Jessie_gemma-3-27b-pt_e1_merged.png filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

KETI_b1_s4_e1_training_log.log

@@ -0,0 +1,212 @@
+04/01/2025 07:35:02 - INFO  - Train data file: finetuning_data_25_sentences.json
+04/01/2025 07:35:02 - INFO  - Output Directory: output/gemma-3-27b-pt/20250401_073446_gemma-3-27b-pt_LoRA
+04/01/2025 07:35:02 - INFO  - Experiment name: KETI_b1_s4_e1
+04/01/2025 07:35:02 - INFO  - torch_dtype: torch.bfloat16
+04/01/2025 07:35:02 - INFO  - 🔍 Start inference on base model: google/gemma-3-27b-it
+04/01/2025 07:35:57 - INFO  - ✅ base_model과 tokenizer 메모리에서 해제 완료
+04/01/2025 07:35:57 - INFO  - Using 6 GPU(s): NVIDIA A100-SXM4-80GB
+04/01/2025 07:35:57 - INFO  - 🔢 Training samples: 37822
+04/01/2025 07:35:57 - INFO  - 🔍 Evaluation samples: 4203
+04/01/2025 07:35:57 - INFO  - 📊 Steps per epoch: 1575
+04/01/2025 07:35:57 - INFO  - 🪜 Total training steps: 1575
+04/01/2025 07:35:57 - INFO  - ✅ FFT or LoRA 모드로 학습합니다.
+04/01/2025 07:36:11 - INFO  - Initializing LORA model...
+04/01/2025 07:36:12 - INFO  - 📌 LoRA Configuration:
+04/01/2025 07:36:12 - INFO  -   - task_type: CAUSAL_LM
+04/01/2025 07:36:12 - INFO  -   - peft_type: PeftType.LORA
+04/01/2025 07:36:12 - INFO  -   - auto_mapping: None
+04/01/2025 07:36:12 - INFO  -   - base_model_name_or_path: google/gemma-3-27b-pt
+04/01/2025 07:36:12 - INFO  -   - revision: None
+04/01/2025 07:36:12 - INFO  -   - inference_mode: False
+04/01/2025 07:36:12 - INFO  -   - r: 16
+04/01/2025 07:36:12 - INFO  -   - target_modules: {'down_proj', 'v_proj', 'k_proj', 'gate_proj', 'up_proj', 'q_proj', 'o_proj'}
+04/01/2025 07:36:12 - INFO  -   - exclude_modules: None
+04/01/2025 07:36:12 - INFO  -   - lora_alpha: 16
+04/01/2025 07:36:12 - INFO  -   - lora_dropout: 0.05
+04/01/2025 07:36:12 - INFO  -   - fan_in_fan_out: False
+04/01/2025 07:36:12 - INFO  -   - bias: none
+04/01/2025 07:36:12 - INFO  -   - use_rslora: False
+04/01/2025 07:36:12 - INFO  -   - modules_to_save: None
+04/01/2025 07:36:12 - INFO  -   - init_lora_weights: True
+04/01/2025 07:36:12 - INFO  -   - layers_to_transform: None
+04/01/2025 07:36:12 - INFO  -   - layers_pattern: None
+04/01/2025 07:36:12 - INFO  -   - rank_pattern: {}
+04/01/2025 07:36:12 - INFO  -   - alpha_pattern: {}
+04/01/2025 07:36:12 - INFO  -   - megatron_config: None
+04/01/2025 07:36:12 - INFO  -   - megatron_core: megatron.core
+04/01/2025 07:36:12 - INFO  -   - trainable_token_indices: None
+04/01/2025 07:36:12 - INFO  -   - loftq_config: {}
+04/01/2025 07:36:12 - INFO  -   - eva_config: None
+04/01/2025 07:36:12 - INFO  -   - corda_config: None
+04/01/2025 07:36:12 - INFO  -   - use_dora: False
+04/01/2025 07:36:12 - INFO  -   - layer_replication: None
+04/01/2025 07:36:12 - INFO  -   - lora_bias: False
+04/01/2025 07:36:12 - INFO  - 🧠 Trainable params: 113516544 / 27122862848 (0.42%)
+04/01/2025 07:36:12 - INFO  - 📌 SFT Configuration:
+04/01/2025 07:36:12 - INFO  -   - output_dir: output/gemma-3-27b-pt/20250401_073446_gemma-3-27b-pt_LoRA
+04/01/2025 07:36:12 - INFO  -   - overwrite_output_dir: False
+04/01/2025 07:36:12 - INFO  -   - do_train: False
+04/01/2025 07:36:12 - INFO  -   - do_eval: False
+04/01/2025 07:36:12 - INFO  -   - do_predict: False
+04/01/2025 07:36:12 - INFO  -   - eval_strategy: no
+04/01/2025 07:36:12 - INFO  -   - prediction_loss_only: False
+04/01/2025 07:36:12 - INFO  -   - per_device_train_batch_size: 1
+04/01/2025 07:36:12 - INFO  -   - per_device_eval_batch_size: 8
+04/01/2025 07:36:12 - INFO  -   - per_gpu_train_batch_size: None
+04/01/2025 07:36:12 - INFO  -   - per_gpu_eval_batch_size: None
+04/01/2025 07:36:12 - INFO  -   - gradient_accumulation_steps: 4
+04/01/2025 07:36:12 - INFO  -   - eval_accumulation_steps: None
+04/01/2025 07:36:12 - INFO  -   - eval_delay: 0
+04/01/2025 07:36:12 - INFO  -   - torch_empty_cache_steps: None
+04/01/2025 07:36:12 - INFO  -   - learning_rate: 0.0002
+04/01/2025 07:36:12 - INFO  -   - weight_decay: 0.0
+04/01/2025 07:36:12 - INFO  -   - adam_beta1: 0.9
+04/01/2025 07:36:12 - INFO  -   - adam_beta2: 0.999
+04/01/2025 07:36:12 - INFO  -   - adam_epsilon: 1e-08
+04/01/2025 07:36:12 - INFO  -   - max_grad_norm: 0.3
+04/01/2025 07:36:12 - INFO  -   - num_train_epochs: 1
+04/01/2025 07:36:12 - INFO  -   - max_steps: -1
+04/01/2025 07:36:12 - INFO  -   - lr_scheduler_type: constant
+04/01/2025 07:36:12 - INFO  -   - lr_scheduler_kwargs: {}
+04/01/2025 07:36:12 - INFO  -   - warmup_ratio: 0.03
+04/01/2025 07:36:12 - INFO  -   - warmup_steps: 0
+04/01/2025 07:36:12 - INFO  -   - log_level: passive
+04/01/2025 07:36:12 - INFO  -   - log_level_replica: warning
+04/01/2025 07:36:12 - INFO  -   - log_on_each_node: True
+04/01/2025 07:36:12 - INFO  -   - logging_dir: output/gemma-3-27b-pt/20250401_073446_gemma-3-27b-pt_LoRA/runs/Apr01_07-36-12_llm-server-779876f58-9zzqd
+04/01/2025 07:36:12 - INFO  -   - logging_strategy: steps
+04/01/2025 07:36:12 - INFO  -   - logging_first_step: False
+04/01/2025 07:36:12 - INFO  -   - logging_steps: 10
+04/01/2025 07:36:12 - INFO  -   - logging_nan_inf_filter: True
+04/01/2025 07:36:12 - INFO  -   - save_strategy: epoch
+04/01/2025 07:36:12 - INFO  -   - save_steps: 500
+04/01/2025 07:36:12 - INFO  -   - save_total_limit: None
+04/01/2025 07:36:12 - INFO  -   - save_safetensors: True
+04/01/2025 07:36:12 - INFO  -   - save_on_each_node: False
+04/01/2025 07:36:12 - INFO  -   - save_only_model: False
+04/01/2025 07:36:12 - INFO  -   - restore_callback_states_from_checkpoint: False
+04/01/2025 07:36:12 - INFO  -   - no_cuda: False
+04/01/2025 07:36:12 - INFO  -   - use_cpu: False
+04/01/2025 07:36:12 - INFO  -   - use_mps_device: False
+04/01/2025 07:36:12 - INFO  -   - seed: 42
+04/01/2025 07:36:12 - INFO  -   - data_seed: None
+04/01/2025 07:36:12 - INFO  -   - jit_mode_eval: False
+04/01/2025 07:36:12 - INFO  -   - use_ipex: False
+04/01/2025 07:36:12 - INFO  -   - bf16: True
+04/01/2025 07:36:12 - INFO  -   - fp16: False
+04/01/2025 07:36:12 - INFO  -   - fp16_opt_level: O1
+04/01/2025 07:36:12 - INFO  -   - half_precision_backend: auto
+04/01/2025 07:36:12 - INFO  -   - bf16_full_eval: False
+04/01/2025 07:36:12 - INFO  -   - fp16_full_eval: False
+04/01/2025 07:36:12 - INFO  -   - tf32: None
+04/01/2025 07:36:12 - INFO  -   - local_rank: 0
+04/01/2025 07:36:12 - INFO  -   - ddp_backend: None
+04/01/2025 07:36:12 - INFO  -   - tpu_num_cores: None
+04/01/2025 07:36:12 - INFO  -   - tpu_metrics_debug: False
+04/01/2025 07:36:12 - INFO  -   - debug: []
+04/01/2025 07:36:12 - INFO  -   - dataloader_drop_last: False
+04/01/2025 07:36:12 - INFO  -   - eval_steps: None
+04/01/2025 07:36:12 - INFO  -   - dataloader_num_workers: 0
+04/01/2025 07:36:12 - INFO  -   - dataloader_prefetch_factor: None
+04/01/2025 07:36:12 - INFO  -   - past_index: -1
+04/01/2025 07:36:12 - INFO  -   - run_name: output/gemma-3-27b-pt/20250401_073446_gemma-3-27b-pt_LoRA
+04/01/2025 07:36:12 - INFO  -   - disable_tqdm: False
+04/01/2025 07:36:12 - INFO  -   - remove_unused_columns: True
+04/01/2025 07:36:12 - INFO  -   - label_names: ['labels']
+04/01/2025 07:36:12 - INFO  -   - load_best_model_at_end: False
+04/01/2025 07:36:12 - INFO  -   - metric_for_best_model: None
+04/01/2025 07:36:12 - INFO  -   - greater_is_better: None
+04/01/2025 07:36:12 - INFO  -   - ignore_data_skip: False
+04/01/2025 07:36:12 - INFO  -   - fsdp: []
+04/01/2025 07:36:12 - INFO  -   - fsdp_min_num_params: 0
+04/01/2025 07:36:12 - INFO  -   - fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
+04/01/2025 07:36:12 - INFO  -   - tp_size: 0
+04/01/2025 07:36:12 - INFO  -   - fsdp_transformer_layer_cls_to_wrap: None
+04/01/2025 07:36:12 - INFO  -   - accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
+04/01/2025 07:36:12 - INFO  -   - deepspeed: None
+04/01/2025 07:36:12 - INFO  -   - label_smoothing_factor: 0.0
+04/01/2025 07:36:12 - INFO  -   - optim: adamw_torch_fused
+04/01/2025 07:36:12 - INFO  -   - optim_args: None
+04/01/2025 07:36:12 - INFO  -   - adafactor: False
+04/01/2025 07:36:12 - INFO  -   - group_by_length: False
+04/01/2025 07:36:12 - INFO  -   - length_column_name: length
+04/01/2025 07:36:12 - INFO  -   - report_to: ['tensorboard']
+04/01/2025 07:36:12 - INFO  -   - ddp_find_unused_parameters: None
+04/01/2025 07:36:12 - INFO  -   - ddp_bucket_cap_mb: None
+04/01/2025 07:36:12 - INFO  -   - ddp_broadcast_buffers: None
+04/01/2025 07:36:12 - INFO  -   - dataloader_pin_memory: True
+04/01/2025 07:36:12 - INFO  -   - dataloader_persistent_workers: False
+04/01/2025 07:36:12 - INFO  -   - skip_memory_metrics: True
+04/01/2025 07:36:12 - INFO  -   - use_legacy_prediction_loop: False
+04/01/2025 07:36:12 - INFO  -   - push_to_hub: True
+04/01/2025 07:36:12 - INFO  -   - resume_from_checkpoint: None
+04/01/2025 07:36:12 - INFO  -   - hub_model_id: None
+04/01/2025 07:36:12 - INFO  -   - hub_strategy: every_save
+04/01/2025 07:36:12 - INFO  -   - hub_token: <HUB_TOKEN>
+04/01/2025 07:36:12 - INFO  -   - hub_private_repo: None
+04/01/2025 07:36:12 - INFO  -   - hub_always_push: False
+04/01/2025 07:36:12 - INFO  -   - gradient_checkpointing: False
+04/01/2025 07:36:12 - INFO  -   - gradient_checkpointing_kwargs: None
+04/01/2025 07:36:12 - INFO  -   - include_inputs_for_metrics: False
+04/01/2025 07:36:12 - INFO  -   - include_for_metrics: []
+04/01/2025 07:36:12 - INFO  -   - eval_do_concat_batches: True
+04/01/2025 07:36:12 - INFO  -   - fp16_backend: auto
+04/01/2025 07:36:12 - INFO  -   - evaluation_strategy: None
+04/01/2025 07:36:12 - INFO  -   - push_to_hub_model_id: None
+04/01/2025 07:36:12 - INFO  -   - push_to_hub_organization: None
+04/01/2025 07:36:12 - INFO  -   - push_to_hub_token: <PUSH_TO_HUB_TOKEN>
+04/01/2025 07:36:12 - INFO  -   - mp_parameters: 
+04/01/2025 07:36:12 - INFO  -   - auto_find_batch_size: False
+04/01/2025 07:36:12 - INFO  -   - full_determinism: False
+04/01/2025 07:36:12 - INFO  -   - torchdynamo: None
+04/01/2025 07:36:12 - INFO  -   - ray_scope: last
+04/01/2025 07:36:12 - INFO  -   - ddp_timeout: 1800
+04/01/2025 07:36:12 - INFO  -   - torch_compile: False
+04/01/2025 07:36:12 - INFO  -   - torch_compile_backend: None
+04/01/2025 07:36:12 - INFO  -   - torch_compile_mode: None
+04/01/2025 07:36:12 - INFO  -   - dispatch_batches: None
+04/01/2025 07:36:12 - INFO  -   - split_batches: None
+04/01/2025 07:36:12 - INFO  -   - include_tokens_per_second: False
+04/01/2025 07:36:12 - INFO  -   - include_num_input_tokens_seen: False
+04/01/2025 07:36:12 - INFO  -   - neftune_noise_alpha: None
+04/01/2025 07:36:12 - INFO  -   - optim_target_modules: None
+04/01/2025 07:36:12 - INFO  -   - batch_eval_metrics: False
+04/01/2025 07:36:12 - INFO  -   - eval_on_start: False
+04/01/2025 07:36:12 - INFO  -   - use_liger_kernel: False
+04/01/2025 07:36:12 - INFO  -   - eval_use_gather_object: False
+04/01/2025 07:36:12 - INFO  -   - average_tokens_across_devices: False
+04/01/2025 07:36:12 - INFO  -   - model_init_kwargs: None
+04/01/2025 07:36:12 - INFO  -   - dataset_text_field: text
+04/01/2025 07:36:12 - INFO  -   - dataset_kwargs: {'add_special_tokens': False, 'append_concat_token': True}
+04/01/2025 07:36:12 - INFO  -   - dataset_num_proc: None
+04/01/2025 07:36:12 - INFO  -   - max_length: 512
+04/01/2025 07:36:12 - INFO  -   - packing: True
+04/01/2025 07:36:12 - INFO  -   - padding_free: False
+04/01/2025 07:36:12 - INFO  -   - eval_packing: None
+04/01/2025 07:36:12 - INFO  -   - dataset_batch_size: None
+04/01/2025 07:36:12 - INFO  -   - num_of_sequences: None
+04/01/2025 07:36:12 - INFO  -   - chars_per_token: <CHARS_PER_TOKEN>
+04/01/2025 07:36:12 - INFO  -   - max_seq_length: 512
+04/01/2025 07:36:12 - INFO  -   - use_liger: None
+04/01/2025 07:36:17 - INFO  - gcc -pthread -B /root/pai/envs/llm-finetuning/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -c /tmp/tmpwl_ntrt_/test.c -o /tmp/tmpwl_ntrt_/test.o
+04/01/2025 07:36:17 - INFO  - gcc -pthread -B /root/pai/envs/llm-finetuning/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -O2 -isystem /root/pai/envs/llm-finetuning/include -fPIC -c /tmp/tmpmhclhoyr/test.c -o /tmp/tmpmhclhoyr/test.o
+04/01/2025 07:36:18 - INFO  - Start Training !
+04/01/2025 07:36:45 - INFO  - [Epoch 0.11] [Step 10] loss: 3.5811
+04/01/2025 07:37:09 - INFO  - [Epoch 0.22] [Step 20] loss: 3.0374
+04/01/2025 07:37:31 - INFO  - [Epoch 0.33] [Step 30] loss: 2.9365
+04/01/2025 07:37:55 - INFO  - [Epoch 0.44] [Step 40] loss: 2.9004
+04/01/2025 07:38:17 - INFO  - [Epoch 0.55] [Step 50] loss: 2.8830
+04/01/2025 07:38:40 - INFO  - [Epoch 0.66] [Step 60] loss: 2.8750
+04/01/2025 07:39:03 - INFO  - [Epoch 0.77] [Step 70] loss: 2.8626
+04/01/2025 07:39:26 - INFO  - [Epoch 0.88] [Step 80] loss: 2.8464
+04/01/2025 07:39:49 - INFO  - [Epoch 0.99] [Step 90] loss: 2.8483
+04/01/2025 07:40:21 - INFO  - ✅ Training complete. Logging system usage...
+04/01/2025 07:40:21 - INFO  - >> System Usage - CPU: 2.5%, RAM: 2.6%, SSD: 75.64GB / 1888.43GB
+04/01/2025 07:40:21 - INFO  - >> GPU 0: 75.38 GB used
+04/01/2025 07:40:21 - INFO  - >> GPU 1: 73.42 GB used
+04/01/2025 07:40:21 - INFO  - >> GPU 2: 75.78 GB used
+04/01/2025 07:40:21 - INFO  - >> GPU 3: 76.61 GB used
+04/01/2025 07:40:21 - INFO  - >> GPU 4: 73.38 GB used
+04/01/2025 07:40:21 - INFO  - >> GPU 5: 73.32 GB used
+04/01/2025 07:40:21 - INFO  - >> Total GPU Memory Used: 447.90 GB
+04/01/2025 07:40:21 - INFO  - >> Total GPU Power Consumption: 532.15 W

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-27b-pt",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "v_proj",
+    "k_proj",
+    "gate_proj",
+    "up_proj",
+    "q_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cacb7d4aad9941c12b6b4abbb87e51d6944f71c8dc5c60b691e09763e008ec3
+size 454183400

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

runs/Apr01_07-36-12_llm-server-779876f58-9zzqd/events.out.tfevents.1743492979.llm-server-779876f58-9zzqd.7519.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19ed6a45a0a4d97d2a3583252df355a77dba0cccc20c15560df6e9544af429ab
+size 9244

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88afbaba67701c9ed80a2f182903fbe0440b0c91b363a6d89db13d7dd49df00a
+size 5816