Upload folder using huggingface_hub

README.md

@@ -0,0 +1,70 @@
+---
+library_name: transformers
+license: apache-2.0
+base_model: dandelin/vilt-b32-finetuned-nlvr2
+tags:
+- image-text-classification
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: '42'
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 42
+
+This model is a fine-tuned version of [dandelin/vilt-b32-finetuned-nlvr2](https://huggingface.co/dandelin/vilt-b32-finetuned-nlvr2) on the nlvr2 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9443
+- Accuracy: 0.7327
+- Dt Accuracy: 0.7327
+- Df Accuracy: 0.4959
+- Unlearn Overall Accuracy: 0.8978
+- Unlearn Time: None
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 32
+- eval_batch_size: 64
+- seed: 42
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Accuracy | Overall Accuracy | Unlearn Overall Accuracy | Time |
+|:-------------:|:-----:|:-----:|:---------------:|:--------:|:----------------:|:------------------------:|:----:|
+| 0.3079        | 1.0   | 2700  | 0.5663          | 0.8542   | 0.6449           | 0.6449                   | None |
+| 0.2432        | 2.0   | 5400  | 0.6403          | 0.7017   | 0.7676           | 0.7676                   | None |
+| 0.1905        | 3.0   | 8100  | 0.7980          | 0.5862   | 0.8458           | 0.8458                   | None |
+| 0.1569        | 4.0   | 10800 | 0.8834          | 0.5176   | 0.8863           | 0.8863                   | None |
+| 0.1273        | 5.0   | 13500 | 0.9443          | 0.4959   | 0.8978           | 0.8978                   | None |
+
+
+### Framework versions
+
+- Transformers 4.48.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.18.0
+- Tokenizers 0.21.0

all_results.json

@@ -0,0 +1,22 @@
+{
+    "df_accuracy": 0.49593967517401394,
+    "df_loss": 1.892191767692566,
+    "dr_accuracy": 0.9968333333333333,
+    "dr_loss": 0.02866685576736927,
+    "dt_accuracy": 0.7440792306588202,
+    "epoch": 5.0,
+    "eval_unlearn_overall_accuracy": 0.8977844146136806,
+    "knowledge_gap": 0.7504383217324052,
+    "ood_accuracy": 0.7559540889526543,
+    "ood_loss": 0.8654005527496338,
+    "test_accuracy": 0.7440792306588202,
+    "test_loss": 0.9116904735565186,
+    "total_flos": 8453448251631240.0,
+    "train_loss": 0.20511870829264323,
+    "train_runtime": 11255.3092,
+    "train_samples_per_second": 38.37,
+    "train_steps_per_second": 1.199,
+    "unlearn_overall_accuracy": 0.9026347180426603,
+    "unlearn_time": 11256.28268122673,
+    "zrf": 0.8546327662809815
+}

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "dandelin/vilt-b32-finetuned-nlvr2",
+  "architectures": [
+    "ViltForImagesAndTextClassification"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "False",
+    "1": "True"
+  },
+  "image_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "False": 0,
+    "True": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "max_image_length": -1,
+  "max_position_embeddings": 40,
+  "modality_type_vocab_size": 3,
+  "model_type": "vilt",
+  "num_attention_heads": 12,
+  "num_channels": 3,
+  "num_hidden_layers": 12,
+  "num_images": 2,
+  "patch_size": 32,
+  "qkv_bias": true,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.48.0",
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

df_results.json

@@ -0,0 +1,4 @@
+{
+    "df_accuracy": 0.49593967517401394,
+    "df_loss": 1.892191767692566
+}

dr_results.json

@@ -0,0 +1,4 @@
+{
+    "dr_accuracy": 0.9968333333333333,
+    "dr_loss": 0.02866685576736927
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9739c21920e00d85ac9f6d8733f06ca10d3790f1f0fdd6f5f30e985b6472836
+size 455875832

ood_results.json

@@ -0,0 +1,4 @@
+{
+    "ood_accuracy": 0.7559540889526543,
+    "ood_loss": 0.8654005527496338
+}

pred_logit_df.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:304aa91662f5dfb030a91382e3ca642293a96dc680b84089b1e8947c5ca06c3f
+size 41504

pred_logit_dr.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1ace116a1cae36e18cd8e2fa379d2be939e624df8673df3f6fc0ba95a5f847
+size 48128

pred_logit_eval.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82a8cdbb7a2e993748265d79ecad4bfdb1237e4c226ef4aeadb5980f69bc0049
+size 55984

pred_logit_ood.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb43d998bcff0c7c81ef3aa0c4daa79a3e8d7044f08bf53fadf1d81fdf27adab
+size 55888

pred_logit_test.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce0a01622d323f272960fc4691beeea35bc7b3a4ccfec25da9c9e9d2446cd57
+size 55864

sal_mask_with_0.5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bacd1deb9b7898b1d187d06515e10fbe2d20eb364a12b26f303ca81d9a92662b
+size 911772538

test_results.json

@@ -0,0 +1,4 @@
+{
+    "test_accuracy": 0.7440792306588202,
+    "test_loss": 0.9116904735565186
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "total_flos": 8453448251631240.0,
+    "train_loss": 0.20511870829264323,
+    "train_runtime": 11255.3092,
+    "train_samples_per_second": 38.37,
+    "train_steps_per_second": 1.199
+}

trainer_state.json

@@ -0,0 +1,301 @@
+{
+  "best_metric": 0.8977844146136806,
+  "best_model_checkpoint": "../../checkpoint/unlearn/nlvr2/vilt/salun/6.0/42/checkpoint-13500",
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 13500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.18518518518518517,
+      "grad_norm": 3.2584547996520996,
+      "learning_rate": 9.62962962962963e-06,
+      "loss": 0.3282,
+      "step": 500
+    },
+    {
+      "epoch": 0.37037037037037035,
+      "grad_norm": 4.812547206878662,
+      "learning_rate": 9.25925925925926e-06,
+      "loss": 0.3197,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5555555555555556,
+      "grad_norm": 4.050705909729004,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 0.3219,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7407407407407407,
+      "grad_norm": 3.562553882598877,
+      "learning_rate": 8.518518518518519e-06,
+      "loss": 0.3194,
+      "step": 2000
+    },
+    {
+      "epoch": 0.9259259259259259,
+      "grad_norm": 5.053567409515381,
+      "learning_rate": 8.148148148148148e-06,
+      "loss": 0.3079,
+      "step": 2500
+    },
+    {
+      "df_accuracy": 0.854215003866976,
+      "dt_accuracy": 0.7513606416499571,
+      "epoch": 1.0,
+      "eval_accuracy": 0.7513606416499571,
+      "eval_loss": 0.5662743449211121,
+      "eval_runtime": 113.7346,
+      "eval_samples_per_second": 61.389,
+      "eval_steps_per_second": 0.967,
+      "eval_unlearn_overall_accuracy": 0.6449160678015313,
+      "step": 2700,
+      "unlearn_overall_accuracy": 0.6449160678015313,
+      "unlearn_time": null
+    },
+    {
+      "epoch": 1.1111111111111112,
+      "grad_norm": 1.7998104095458984,
+      "learning_rate": 7.77777777777778e-06,
+      "loss": 0.2708,
+      "step": 3000
+    },
+    {
+      "epoch": 1.2962962962962963,
+      "grad_norm": 2.322847366333008,
+      "learning_rate": 7.4074074074074075e-06,
+      "loss": 0.2454,
+      "step": 3500
+    },
+    {
+      "epoch": 1.4814814814814814,
+      "grad_norm": 2.8383140563964844,
+      "learning_rate": 7.0370370370370375e-06,
+      "loss": 0.2361,
+      "step": 4000
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "grad_norm": 2.5046894550323486,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.2486,
+      "step": 4500
+    },
+    {
+      "epoch": 1.8518518518518519,
+      "grad_norm": 4.192043781280518,
+      "learning_rate": 6.296296296296297e-06,
+      "loss": 0.2432,
+      "step": 5000
+    },
+    {
+      "df_accuracy": 0.7016627996906419,
+      "dt_accuracy": 0.7433400171870524,
+      "epoch": 2.0,
+      "eval_accuracy": 0.7433400171870524,
+      "eval_loss": 0.6402536630630493,
+      "eval_runtime": 108.1015,
+      "eval_samples_per_second": 64.587,
+      "eval_steps_per_second": 1.018,
+      "eval_unlearn_overall_accuracy": 0.7676276016043365,
+      "step": 5400,
+      "unlearn_overall_accuracy": 0.7676276016043365,
+      "unlearn_time": null
+    },
+    {
+      "epoch": 2.037037037037037,
+      "grad_norm": 4.436009407043457,
+      "learning_rate": 5.925925925925926e-06,
+      "loss": 0.2304,
+      "step": 5500
+    },
+    {
+      "epoch": 2.2222222222222223,
+      "grad_norm": 3.5263144969940186,
+      "learning_rate": 5.555555555555557e-06,
+      "loss": 0.1807,
+      "step": 6000
+    },
+    {
+      "epoch": 2.4074074074074074,
+      "grad_norm": 3.2321693897247314,
+      "learning_rate": 5.185185185185185e-06,
+      "loss": 0.188,
+      "step": 6500
+    },
+    {
+      "epoch": 2.5925925925925926,
+      "grad_norm": 3.6127853393554688,
+      "learning_rate": 4.814814814814815e-06,
+      "loss": 0.1931,
+      "step": 7000
+    },
+    {
+      "epoch": 2.7777777777777777,
+      "grad_norm": 4.856908798217773,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 0.1923,
+      "step": 7500
+    },
+    {
+      "epoch": 2.962962962962963,
+      "grad_norm": 8.728678703308105,
+      "learning_rate": 4.074074074074074e-06,
+      "loss": 0.1905,
+      "step": 8000
+    },
+    {
+      "df_accuracy": 0.5862335653518949,
+      "dt_accuracy": 0.7396161558292753,
+      "epoch": 3.0,
+      "eval_accuracy": 0.7396161558292753,
+      "eval_loss": 0.7980125546455383,
+      "eval_runtime": 102.947,
+      "eval_samples_per_second": 67.821,
+      "eval_steps_per_second": 1.069,
+      "eval_unlearn_overall_accuracy": 0.845758066951303,
+      "step": 8100,
+      "unlearn_overall_accuracy": 0.845758066951303,
+      "unlearn_time": null
+    },
+    {
+      "epoch": 3.148148148148148,
+      "grad_norm": 0.8636759519577026,
+      "learning_rate": 3.7037037037037037e-06,
+      "loss": 0.1548,
+      "step": 8500
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 4.8238606452941895,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 0.1483,
+      "step": 9000
+    },
+    {
+      "epoch": 3.5185185185185186,
+      "grad_norm": 3.636707067489624,
+      "learning_rate": 2.962962962962963e-06,
+      "loss": 0.1477,
+      "step": 9500
+    },
+    {
+      "epoch": 3.7037037037037037,
+      "grad_norm": 0.8585140705108643,
+      "learning_rate": 2.5925925925925925e-06,
+      "loss": 0.1434,
+      "step": 10000
+    },
+    {
+      "epoch": 3.888888888888889,
+      "grad_norm": 3.986074209213257,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 0.1569,
+      "step": 10500
+    },
+    {
+      "df_accuracy": 0.5175947409126064,
+      "dt_accuracy": 0.7354626181609853,
+      "epoch": 4.0,
+      "eval_accuracy": 0.7354626181609853,
+      "eval_loss": 0.8833993673324585,
+      "eval_runtime": 109.8215,
+      "eval_samples_per_second": 63.576,
+      "eval_steps_per_second": 1.002,
+      "eval_unlearn_overall_accuracy": 0.8863248174250627,
+      "step": 10800,
+      "unlearn_overall_accuracy": 0.8863248174250627,
+      "unlearn_time": null
+    },
+    {
+      "epoch": 4.074074074074074,
+      "grad_norm": 2.4611103534698486,
+      "learning_rate": 1.8518518518518519e-06,
+      "loss": 0.1421,
+      "step": 11000
+    },
+    {
+      "epoch": 4.2592592592592595,
+      "grad_norm": 3.0824742317199707,
+      "learning_rate": 1.4814814814814815e-06,
+      "loss": 0.1283,
+      "step": 11500
+    },
+    {
+      "epoch": 4.444444444444445,
+      "grad_norm": 2.9113352298736572,
+      "learning_rate": 1.111111111111111e-06,
+      "loss": 0.1198,
+      "step": 12000
+    },
+    {
+      "epoch": 4.62962962962963,
+      "grad_norm": 0.5261613130569458,
+      "learning_rate": 7.407407407407407e-07,
+      "loss": 0.1196,
+      "step": 12500
+    },
+    {
+      "epoch": 4.814814814814815,
+      "grad_norm": 4.700664520263672,
+      "learning_rate": 3.7037037037037036e-07,
+      "loss": 0.1336,
+      "step": 13000
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 4.13147497177124,
+      "learning_rate": 0.0,
+      "loss": 0.1273,
+      "step": 13500
+    },
+    {
+      "df_accuracy": 0.49593967517401394,
+      "dt_accuracy": 0.7327413348610713,
+      "epoch": 5.0,
+      "eval_accuracy": 0.7327413348610713,
+      "eval_loss": 0.9442616701126099,
+      "eval_runtime": 113.4763,
+      "eval_samples_per_second": 61.528,
+      "eval_steps_per_second": 0.969,
+      "eval_unlearn_overall_accuracy": 0.8977844146136806,
+      "step": 13500,
+      "unlearn_overall_accuracy": 0.8977844146136806,
+      "unlearn_time": null
+    },
+    {
+      "epoch": 5.0,
+      "step": 13500,
+      "total_flos": 8453448251631240.0,
+      "train_loss": 0.20511870829264323,
+      "train_runtime": 11255.3092,
+      "train_samples_per_second": 38.37,
+      "train_steps_per_second": 1.199
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 13500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8453448251631240.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81965d692c249bb4108c107b1b67353e93c98fba6cdf56f792b8e784b024ba74
+size 5432

unlearn_final_results.json

@@ -0,0 +1,11 @@
+{
+    "df_accuracy": 0.49593967517401394,
+    "dr_accuracy": 0.9968333333333333,
+    "dt_accuracy": 0.7440792306588202,
+    "eval_unlearn_overall_accuracy": 0.8977844146136806,
+    "knowledge_gap": 0.7504383217324052,
+    "ood_accuracy": 0.7559540889526543,
+    "unlearn_overall_accuracy": 0.9026347180426603,
+    "unlearn_time": 11256.28268122673,
+    "zrf": 0.8546327662809815
+}