config.json

{
  "architectures": [
    "ILLUMEForConditionalGeneration"
  ],
  "auto_map": {
    "AutoConfig": "configuration_illume.ILLUMEConfig",
    "AutoModel": "modeling_illume.ILLUMEForConditionalGeneration",
    "AutoModelForCausalLM": "modeling_illume.ILLUMEForConditionalGeneration"
  },
  "ignore_index": -100,
  "image_out_token_index": 282777,
  "image_token_index": 282776,
  "mm_projector_config": {
    "hidden_size": 3584,
    "mm_hidden_size": [
      3584,
      32
    ],
    "projector_cfg1": {
      "mlp_depth": 2,
      "type": "MLPProjector"
    },
    "projector_cfg2": {
      "mlp_depth": 2,
      "type": "MLPProjector"
    },
    "trainable": true,
    "type": "MixedProjector"
  },
  "model_type": "illume",
  "special_tokens_ids": {
    "<end_of_image>": 151666,
    "<end_of_level0>": 151669,
    "<end_of_level1>": 151671,
    "<end_of_line>": 151667,
    "<start_of_image>": 151665,
    "<start_of_level0>": 151668,
    "<start_of_level1>": 151670
  },
  "text_config": {
    "_name_or_path": "./Qwen2.5-7B-Instruct-with-vision-tokenizer-32k-96k-level2",
    "architectures": [
      "Qwen2ForCausalLM"
    ],
    "bos_token_id": 151643,
    "eos_token_id": 151645,
    "hidden_size": 3584,
    "intermediate_size": 18944,
    "model_type": "qwen2",
    "num_attention_heads": 28,
    "num_hidden_layers": 28,
    "num_key_value_heads": 4,
    "rope_theta": 1000000.0,
    "torch_dtype": "bfloat16",
    "vocab_size": 283175
  },
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.44.2",
  "vision_config": {
    "_name_or_path": "./dualvitok/",
    "architectures": [
      "DualViTok"
    ],
    "auto_map": {
      "AutoConfig": "configuration_dualvitok.DualViTokConfig",
      "AutoModel": "modeling_dualvitok.DualViTok"
    },
    "model_type": "DualViTok",
    "pixel_decoder": {
      "attn_resolutions": [
        4
      ],
      "ch": 384,
      "ch_mult": [
        1,
        1,
        2,
        2,
        4
      ],
      "codebook_size": 98304,
      "embed_dim": 64,
      "use_dc_up_down_blocks": true,
      "z_channels": 64
    },
    "pixel_encoder": {
      "attn_resolutions": [
        4
      ],
      "ch": 128,
      "ch_mult": [
        1,
        1,
        2,
        2,
        4
      ],
      "codebook_size": 98304,
      "embed_dim": 32,
      "use_dc_up_down_blocks": true,
      "z_channels": 32
    },
    "semantic_decoder": {
      "pretrained_semantic_encoder": "Emova-ollm/qwen2vit600m",
      "target_mlp": "norm"
    },
    "semantic_encoder": {
      "pretrained_semantic_encoder": {
        "_name_or_path": "Emova-ollm/qwen2vit600m",
        "add_cross_attention": false,
        "architectures": [
          "Qwen2VisionTransformerPretrainedModel"
        ],
        "bad_words_ids": null,
        "begin_suppress_tokens": null,
        "bos_token_id": null,
        "chunk_size_feed_forward": 0,
        "cross_attention_hidden_size": null,
        "decoder_start_token_id": null,
        "depth": 32,
        "diversity_penalty": 0.0,
        "do_sample": false,
        "early_stopping": false,
        "embed_dim": 1280,
        "encoder_no_repeat_ngram_size": 0,
        "eos_token_id": null,
        "exponential_decay_length_penalty": null,
        "finetuning_task": null,
        "forced_bos_token_id": null,
        "forced_eos_token_id": null,
        "hidden_act": "quick_gelu",
        "hidden_size": 3584,
        "id2label": {
          "0": "LABEL_0",
          "1": "LABEL_1"
        },
        "in_channels": 3,
        "in_chans": 3,
        "initializer_range": 0.02,
        "is_decoder": false,
        "is_encoder_decoder": false,
        "label2id": {
          "LABEL_0": 0,
          "LABEL_1": 1
        },
        "length_penalty": 1.0,
        "max_length": 20,
        "min_length": 0,
        "mlp_ratio": 4,
        "model_type": "qwen2_vl",
        "no_repeat_ngram_size": 0,
        "num_beam_groups": 1,
        "num_beams": 1,
        "num_heads": 16,
        "num_return_sequences": 1,
        "output_attentions": false,
        "output_hidden_states": false,
        "output_scores": false,
        "pad_token_id": null,
        "patch_size": 14,
        "prefix": null,
        "problem_type": null,
        "pruned_heads": {},
        "remove_invalid_values": false,
        "repetition_penalty": 1.0,
        "return_dict": true,
        "return_dict_in_generate": false,
        "sep_token_id": null,
        "spatial_merge_size": 2,
        "spatial_patch_size": 14,
        "suppress_tokens": null,
        "task_specific_params": null,
        "temperature": 1.0,
        "temporal_patch_size": 2,
        "tf_legacy_loss": false,
        "tie_encoder_decoder": false,
        "tie_word_embeddings": true,
        "tokenizer_class": null,
        "top_k": 50,
        "top_p": 1.0,
        "torch_dtype": "float32",
        "torchscript": false,
        "transformers_version": "4.44.2",
        "typical_p": 1.0,
        "use_bfloat16": false
      }
    },
    "torch_dtype": "float16"
  }
}