Add files using upload-large-folder tool

tests/tuners/test_extra_state_dict.py

@@ -0,0 +1,63 @@
+import os.path
+import shutil
+import tempfile
+import unittest
+
+import torch
+from modelscope import Model
+
+from swift import LoRAConfig, Swift
+from swift.tuners.utils import ModulesToSaveWrapper
+
+
+class TestExtraStateDict(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def test_swift_extra_state_dict(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        model = Swift.prepare_model(model, lora_config, extra_state_keys=['classifier.*'])
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'extra_states', 'adapter_model.bin')))
+        state_dict = torch.load(os.path.join(self.tmp_dir, 'extra_states', 'adapter_model.bin'))
+        self.assertTrue(any('classifier' in key for key in state_dict))
+        state_dict['classifier.weight'] = torch.ones_like(state_dict['classifier.weight']) * 2.0
+        with open(os.path.join(self.tmp_dir, 'extra_states', 'adapter_model.bin'), 'wb') as f:
+            torch.save(state_dict, f)
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        model = Swift.from_pretrained(model, self.tmp_dir, inference_mode=False)
+        names = [name for name, value in model.named_parameters() if value.requires_grad]
+        self.assertTrue(any('classifier' in name for name in names))
+        self.assertTrue(torch.allclose(state_dict['classifier.weight'], model.base_model.classifier.weight))
+
+    def test_swift_modules_to_save(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'], modules_to_save=['classifier'])
+        lora_config2 = LoRAConfig(target_modules=['query', 'key', 'value'], modules_to_save=['classifier'])
+        model = Swift.prepare_model(model, {'lora1': lora_config, 'lora2': lora_config2})
+        model.set_active_adapters('lora1')
+        model.set_active_adapters('lora2')
+        self.assertTrue(isinstance(model.classifier, ModulesToSaveWrapper))
+        self.assertTrue(model.classifier.active_adapter == 'lora2')
+        model.save_pretrained(self.tmp_dir)
+        state_dict = torch.load(os.path.join(self.tmp_dir, 'lora2', 'adapter_model.bin'))
+        self.assertTrue(any('classifier' in key for key in state_dict))
+        state_dict['classifier.weight'] = torch.ones_like(state_dict['classifier.weight']) * 2.0
+        with open(os.path.join(self.tmp_dir, 'lora2', 'adapter_model.bin'), 'wb') as f:
+            torch.save(state_dict, f)
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        model = Swift.from_pretrained(model, self.tmp_dir, adapter_name='lora2')
+        names = [name for name, value in model.named_parameters() if value.requires_grad]
+        self.assertTrue(any('classifier' in name for name in names))
+        self.assertTrue(
+            torch.allclose(state_dict['classifier.weight'],
+                           model.base_model.classifier.modules_to_save['lora2'].weight))

tests/tuners/test_merged_linear.py

@@ -0,0 +1,44 @@
+import math
+import unittest
+
+import torch
+from modelscope import Model, Preprocessor
+from torch import nn
+
+from swift import LoRAConfig, Swift
+
+
+class TestMergedLinear(unittest.TestCase):
+
+    def test_swift_lora_forward(self):
+
+        from swift.tuners.lora import MergedLinear
+
+        def reset_parameters(self):
+            nn.Linear.reset_parameters(self)
+            if hasattr(self, 'lora_A'):
+                # initialize A the same way as the default for nn.Linear and B to zero
+                nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
+                nn.init.ones_(self.lora_B)
+
+        MergedLinear.reset_parameters = reset_parameters
+
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        lora_config = LoRAConfig(
+            target_modules=['query', 'key', 'value'], use_merged_linear=True, enable_lora=[True, True, True])
+        outputs = model(**inputs)
+        model = Swift.prepare_model(model, config=lora_config)
+        model.eval()
+        outputs_lora = model(**inputs)
+        model.deactivate_adapter('default')
+        outputs_deactivate = model(**inputs)
+        model.activate_adapter('default')
+        outputs_reactivate = model(**inputs)
+        Swift.merge_and_unload(model)
+        outputs_merged = model(**inputs)
+        self.assertTrue(torch.allclose(outputs.logits, outputs_deactivate.logits))
+        self.assertTrue(not torch.allclose(outputs.logits, outputs_lora.logits))
+        self.assertTrue(torch.allclose(outputs_lora.logits, outputs_reactivate.logits))
+        self.assertTrue(torch.allclose(outputs_lora.logits, outputs_merged.logits, atol=1e-4))

tests/tuners/test_neft.py

@@ -0,0 +1,109 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+import torch
+from modelscope import AutoModel, Preprocessor
+from peft.utils import WEIGHTS_NAME
+from transformers import PreTrainedModel
+
+from swift import LoRAConfig, Swift
+from swift.tuners import NEFTuneConfig
+
+
+class TestNEFT(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def test_neft(self):
+        model = AutoModel.from_pretrained('AI-ModelScope/bert-base-uncased')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        config = NEFTuneConfig()
+
+        t1 = model.embeddings.word_embeddings(inputs['input_ids'])
+        model = Swift.prepare_model(model, config)
+        model.train()
+        t2 = model.embeddings.word_embeddings(inputs['input_ids'])
+        model.deactivate_adapter('default')
+        t3 = model.embeddings.word_embeddings(inputs['input_ids'])
+        self.assertTrue(torch.allclose(t1, t3))
+        self.assertFalse(torch.allclose(t1, t2))
+        model.save_pretrained(self.tmp_dir)
+        bin_file = os.path.join(self.tmp_dir, 'pytorch_model.bin')
+        self.assertTrue(os.path.isfile(bin_file))
+        model2 = AutoModel.from_pretrained(self.tmp_dir)
+
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        self.assertTrue(len(state_dict) > 0)
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+        shutil.rmtree(self.tmp_dir)
+        PreTrainedModel.origin_save_pretrained = PreTrainedModel.save_pretrained
+        delattr(PreTrainedModel, 'save_pretrained')
+        model.save_pretrained(self.tmp_dir)
+        bin_file = os.path.join(self.tmp_dir, WEIGHTS_NAME)
+        self.assertTrue(os.path.isfile(bin_file))
+        model_new = AutoModel.from_pretrained('AI-ModelScope/bert-base-uncased')
+        model_new_2 = Swift.from_pretrained(model_new, self.tmp_dir)
+
+        state_dict = model.state_dict()
+        state_dict2 = model_new_2.state_dict()
+        self.assertTrue(len(state_dict) > 0)
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+        PreTrainedModel.save_pretrained = PreTrainedModel.origin_save_pretrained
+
+    def test_neft_lora(self):
+        model = AutoModel.from_pretrained('AI-ModelScope/bert-base-uncased')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        config = NEFTuneConfig()
+        config2 = LoRAConfig(target_modules=['query', 'key', 'value'])
+
+        t1 = model.embeddings.word_embeddings(inputs['input_ids'])
+        model = Swift.prepare_model(model, {'c1': config, 'c2': config2})
+        model.train()
+        t2 = model.embeddings.word_embeddings(inputs['input_ids'])
+        model.deactivate_adapter('c1')
+        t3 = model.embeddings.word_embeddings(inputs['input_ids'])
+        self.assertTrue(torch.allclose(t1, t3))
+        self.assertFalse(torch.allclose(t1, t2))
+        model.save_pretrained(self.tmp_dir)
+        bin_file = os.path.join(self.tmp_dir, 'c2', WEIGHTS_NAME)
+        self.assertTrue(os.path.isfile(bin_file))
+        bin_file = os.path.join(self.tmp_dir, 'c1', WEIGHTS_NAME)
+        self.assertTrue(not os.path.isfile(bin_file))
+        model_new = AutoModel.from_pretrained('AI-ModelScope/bert-base-uncased')
+        t1 = model_new.embeddings.word_embeddings(inputs['input_ids'])
+        model_new = Swift.from_pretrained(model_new, self.tmp_dir)
+        model_new.train()
+        t2 = model_new.embeddings.word_embeddings(inputs['input_ids'])
+        model_new.eval()
+        t4 = model_new.embeddings.word_embeddings(inputs['input_ids'])
+        model_new.train()
+        model_new.deactivate_adapter('c1')
+        t3 = model_new.embeddings.word_embeddings(inputs['input_ids'])
+        self.assertTrue(torch.allclose(t1, t3))
+        self.assertTrue(torch.allclose(t1, t4))
+        self.assertFalse(torch.allclose(t1, t2))
+
+        state_dict = model.state_dict()
+        state_dict2 = model_new.state_dict()
+        self.assertTrue(len(state_dict) > 0 and all(['lora' in key for key in state_dict.keys()]))
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))

tests/tuners/test_peft.py

@@ -0,0 +1,160 @@
+import copy
+import os
+import shutil
+import tempfile
+import unittest
+
+import peft
+import torch
+from modelscope import Preprocessor
+from modelscope.models.nlp.structbert import SbertConfig, SbertForSequenceClassification
+from peft import PeftModel, inject_adapter_in_model
+from peft.config import PeftConfigMixin
+from peft.tuners.lora import Linear
+from peft.utils import WEIGHTS_NAME
+from torch import nn
+
+from swift import AdaLoraConfig, LoraConfig, LoRAConfig, Swift, get_peft_model
+
+
+class TestPeft(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def test_peft_lora_injection(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        model2 = copy.deepcopy(model)
+        lora_config = LoraConfig(target_modules=['query', 'key', 'value'])
+        model = Swift.prepare_model(model, lora_config)
+        model.save_pretrained(self.tmp_dir, safe_serialization=False)
+        with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f:
+            f.write('{}')
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, WEIGHTS_NAME)))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir)
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+    @unittest.skip
+    def test_lora_merge(self):
+
+        def reset_lora_parameters(self, adapter_name, init_lora_weights):
+            if init_lora_weights is False:
+                return
+
+            if adapter_name == 'default':
+                ratio = 1.0
+            elif adapter_name == 'second':
+                ratio = 2.0
+            else:
+                ratio = 3.0
+
+            if adapter_name in self.lora_A.keys():
+                nn.init.ones_(self.lora_A[adapter_name].weight)
+                self.lora_A[adapter_name].weight.data = self.lora_A[adapter_name].weight.data * ratio
+                nn.init.ones_(self.lora_B[adapter_name].weight)
+
+        Linear.reset_lora_parameters = reset_lora_parameters
+
+        model = SbertForSequenceClassification(SbertConfig())
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        model = Swift.prepare_model(model, lora_config)
+        lora_config2 = LoRAConfig(target_modules=['query', 'key', 'value'])
+        model = Swift.prepare_model(model, {'second': lora_config2})
+        model.add_weighted_adapter(['default', 'second'],
+                                   weights=[0.7, 0.3],
+                                   adapter_name='test',
+                                   combination_type='cat')
+        self.assertTrue(model.base_model.bert.encoder.layer[0].attention.self.key.active_adapter == ['test'])
+
+        model2 = SbertForSequenceClassification(SbertConfig())
+        lora_config = LoraConfig(target_modules=['query', 'key', 'value'])
+        model2 = get_peft_model(model2, lora_config)
+        lora_config2 = LoraConfig(target_modules=['query', 'key', 'value'])
+        inject_adapter_in_model(lora_config2, model2, adapter_name='second')
+        model2.add_weighted_adapter(['default', 'second'],
+                                    weights=[0.7, 0.3],
+                                    adapter_name='test',
+                                    combination_type='cat')
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        state_dict2 = {key[len('base_model.model.'):]: value for key, value in state_dict2.items() if 'lora' in key}
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        print(model(**inputs))
+        model.save_pretrained(self.tmp_dir)
+        model3 = SbertForSequenceClassification(SbertConfig())
+        model3 = Swift.from_pretrained(model3, self.tmp_dir)
+        state_dict3 = model3.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict3)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict3[key]).flatten().detach().cpu()))
+
+    def test_lora_reload_by_peft(self):
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        model = SbertForSequenceClassification(SbertConfig())
+        model2 = copy.deepcopy(model)
+        model = Swift.prepare_model(model, lora_config)
+        model.save_pretrained(self.tmp_dir, peft_format=True)
+        model2 = PeftModel.from_pretrained(model2, self.tmp_dir)
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        state_dict2 = {key[len('base_model.model.'):]: value for key, value in state_dict2.items() if 'lora' in key}
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+    def test_peft_adalora_injection(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        model2 = copy.deepcopy(model)
+        adalora_config = AdaLoraConfig(target_modules=['query', 'key', 'value'], total_step=1)
+        model = Swift.prepare_model(model, adalora_config)
+        model.save_pretrained(self.tmp_dir, safe_serialization=False)
+        with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f:
+            f.write('{}')
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, WEIGHTS_NAME)))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir)
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+    @unittest.skip
+    def test_peft_lora_dtype(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        model2 = copy.deepcopy(model)
+        model3 = copy.deepcopy(model)
+        lora_config = LoraConfig(target_modules=['query', 'key', 'value'], lora_dtype='float16')
+        model = Swift.prepare_model(model, lora_config)
+        model.save_pretrained(self.tmp_dir, safe_serialization=False)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'additional_config.json')))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir)
+        self.assertTrue(model2.base_model.model.bert.encoder.layer[0].attention.self.key.lora_A.default.weight.dtype ==
+                        torch.float16)
+        self.assertTrue(model2.peft_config['default'].lora_dtype == 'float16')
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+        PeftConfigMixin.from_pretrained = PeftConfigMixin.from_pretrained_origin
+        model3 = Swift.from_pretrained(model3, self.tmp_dir)
+        self.assertTrue(model3.base_model.model.bert.encoder.layer[0].attention.self.key.lora_A.default.weight.dtype ==
+                        torch.float32)
+        self.assertTrue(isinstance(model3.peft_config['default'], peft.LoraConfig))

tests/tuners/test_scetuning.py

@@ -0,0 +1,131 @@
+import copy
+import os
+import shutil
+import tempfile
+import unittest
+
+import torch
+from modelscope import snapshot_download
+
+from swift import SCETuningConfig, Swift
+from swift.tuners.part import PartConfig
+
+
+class TestSCETuning(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def model_comparison(self, model, model2):
+        model_key = list(model.state_dict().keys())
+        model2_key = list(model2.state_dict().keys())
+        self.assertTrue(model_key == model2_key)
+        model_val = torch.sum(torch.stack([torch.sum(val) for val in model.state_dict().values()]))
+        model2_val = torch.sum(torch.stack([torch.sum(val) for val in model2.state_dict().values()]))
+        self.assertTrue(torch.isclose(model_val, model2_val))
+
+    def test_scetuning_on_diffusers_v1(self):
+        model_dir = snapshot_download('AI-ModelScope/stable-diffusion-v1-5')
+        from diffusers import UNet2DConditionModel
+        model = UNet2DConditionModel.from_pretrained(model_dir, subfolder='unet')
+        model.requires_grad_(False)
+        model_check = copy.deepcopy(model)
+        # module_keys = [key for key, _ in model.named_modules()]
+        scetuning_config = SCETuningConfig(
+            dims=[320, 320, 320, 320, 640, 640, 640, 1280, 1280, 1280, 1280, 1280],
+            tuner_mode='encoder',
+            target_modules=[
+                'conv_in', 'down_blocks.0.attentions.0', 'down_blocks.0.attentions.1', 'down_blocks.0.downsamplers',
+                'down_blocks.1.attentions.0', 'down_blocks.1.attentions.1', 'down_blocks.1.downsamplers',
+                'down_blocks.2.attentions.0', 'down_blocks.2.attentions.1', 'down_blocks.2.downsamplers',
+                'down_blocks.3.resnets.0', 'down_blocks.3.resnets.1'
+            ])
+        model = Swift.prepare_model(model, config=scetuning_config)
+        print(model.get_trainable_parameters())
+        input_data = {
+            'sample': torch.ones((1, 4, 64, 64)),
+            'timestep': 10,
+            'encoder_hidden_states': torch.ones((1, 77, 768))
+        }
+        result = model(**input_data).sample
+        print(result.shape)
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        model_check = Swift.from_pretrained(model_check, self.tmp_dir)
+        self.model_comparison(model, model_check)
+
+    def test_scetuning_part_mixin(self):
+        model_dir = snapshot_download('AI-ModelScope/stable-diffusion-v1-5')
+        from diffusers import UNet2DConditionModel
+        model = UNet2DConditionModel.from_pretrained(model_dir, subfolder='unet')
+        model.requires_grad_(False)
+        model_check = copy.deepcopy(model)
+        # module_keys = [key for key, _ in model.named_modules()]
+        scetuning_config = SCETuningConfig(
+            dims=[320, 320, 320, 320, 640, 640, 640, 1280, 1280, 1280, 1280, 1280],
+            tuner_mode='encoder',
+            target_modules=[
+                'conv_in', 'down_blocks.0.attentions.0', 'down_blocks.0.attentions.1', 'down_blocks.0.downsamplers',
+                'down_blocks.1.attentions.0', 'down_blocks.1.attentions.1', 'down_blocks.1.downsamplers',
+                'down_blocks.2.attentions.0', 'down_blocks.2.attentions.1', 'down_blocks.2.downsamplers',
+                'down_blocks.3.resnets.0', 'down_blocks.3.resnets.1'
+            ])
+        targets = r'.*(to_k|to_v).*'
+        part_config = PartConfig(target_modules=targets)
+        model = Swift.prepare_model(model, config=scetuning_config)
+        model = Swift.prepare_model(model, config={'part': part_config})
+        print(model.get_trainable_parameters())
+        input_data = {
+            'sample': torch.ones((1, 4, 64, 64)),
+            'timestep': 10,
+            'encoder_hidden_states': torch.ones((1, 77, 768))
+        }
+        model.set_active_adapters('default')
+        model.set_active_adapters('part')
+        model.set_active_adapters('default')
+        result = model(**input_data).sample
+        print(result.shape)
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        model_check = Swift.from_pretrained(model_check, self.tmp_dir)
+        self.model_comparison(model, model_check)
+
+    def test_scetuning_on_diffusers_v2(self):
+        model_dir = snapshot_download('AI-ModelScope/stable-diffusion-v1-5')
+        from diffusers import UNet2DConditionModel
+        model = UNet2DConditionModel.from_pretrained(model_dir, subfolder='unet')
+        model.requires_grad_(False)
+        model_check = copy.deepcopy(model)
+        # module_keys = [key for key, _ in model.named_modules()]
+        scetuning_config = SCETuningConfig(
+            dims=[1280, 1280, 1280, 1280, 1280, 640, 640, 640, 320, 320, 320, 320],
+            tuner_mode='decoder',
+            target_modules=[
+                'up_blocks.0.resnets.0', 'up_blocks.0.resnets.1', 'up_blocks.0.resnets.2', 'up_blocks.1.resnets.0',
+                'up_blocks.1.resnets.1', 'up_blocks.1.resnets.2', 'up_blocks.2.resnets.0', 'up_blocks.2.resnets.1',
+                'up_blocks.2.resnets.2', 'up_blocks.3.resnets.0', 'up_blocks.3.resnets.1', 'up_blocks.3.resnets.2'
+            ])
+        model = Swift.prepare_model(model, config=scetuning_config)
+        print(model.get_trainable_parameters())
+        input_data = {
+            'sample': torch.ones((1, 4, 64, 64)),
+            'timestep': 10,
+            'encoder_hidden_states': torch.ones((1, 77, 768))
+        }
+        result = model(**input_data).sample
+        print(result.shape)
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        model_check = Swift.from_pretrained(model_check, self.tmp_dir)
+        self.model_comparison(model, model_check)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/tuners/test_swift_base.py

@@ -0,0 +1,553 @@
+import copy
+import math
+import os
+import re
+import shutil
+import tempfile
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+
+import peft
+import torch
+from modelscope import Model, Preprocessor
+from modelscope.models.nlp.structbert import SbertConfig, SbertForSequenceClassification
+from peft import PeftModel
+from peft.utils import WEIGHTS_NAME
+from torch import nn
+
+from swift import AdapterConfig, LoRAConfig, PromptConfig, ResTuningConfig, SideConfig, Swift, SwiftModel
+from swift.tuners.part import Part, PartConfig
+
+
+class TestSwift(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def test_swift_lora_forward(self):
+
+        from swift.tuners.lora import Linear
+
+        def reset_lora_parameters(self, adapter_name, init_lora_weights):
+            if init_lora_weights is False:
+                return
+
+            if adapter_name in self.lora_A.keys():
+                if init_lora_weights is True:
+                    # initialize A the same way as the default for nn.Linear and B to zero
+                    # https://github.com/microsoft/LoRA/blob/a0a92e0f26c067cf94747bdbf1ce73793fa44d19/loralib/layers.py#L124
+                    nn.init.kaiming_uniform_(self.lora_A[adapter_name].weight, a=math.sqrt(5))
+                elif init_lora_weights.lower() == 'gaussian':
+                    nn.init.normal_(self.lora_A[adapter_name].weight, std=1 / self.r[adapter_name])
+                else:
+                    raise ValueError(f'Unknown initialization {init_lora_weights=}')
+                nn.init.ones_(self.lora_B[adapter_name].weight)
+            if adapter_name in self.lora_embedding_A.keys():
+                # initialize a the same way as the default for nn.linear and b to zero
+                nn.init.ones_(self.lora_embedding_A[adapter_name])
+                nn.init.normal_(self.lora_embedding_B[adapter_name])
+
+        Linear.reset_lora_parameters = reset_lora_parameters
+
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        outputs = model(**inputs)
+        model = Swift.prepare_model(model, config=lora_config)
+        model.eval()
+        outputs_lora = model(**inputs)
+        model.deactivate_adapter('default')
+        outputs_deactivate = model(**inputs)
+        model.activate_adapter('default')
+        outputs_reactivate = model(**inputs)
+        self.assertTrue(torch.allclose(outputs.logits, outputs_deactivate.logits))
+        self.assertTrue(not torch.allclose(outputs.logits, outputs_lora.logits))
+        self.assertTrue(torch.allclose(outputs_lora.logits, outputs_reactivate.logits))
+
+    def test_swift_adapter_forward(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        adapter_config = AdapterConfig(
+            dim=model.config.hidden_size,
+            target_modules=r'.*layer\.\d+$',
+            method_name='feed_forward_chunk',
+            hidden_pos=0)
+        outputs = model(**inputs)
+        model = Swift.prepare_model(model, config=adapter_config)
+        outputs_lora = model(**inputs)
+        model.deactivate_adapter('default')
+        outputs_deactivate = model(**inputs)
+        model.activate_adapter('default')
+        outputs_reactivate = model(**inputs)
+        self.assertTrue(torch.allclose(outputs.logits, outputs_deactivate.logits))
+        self.assertTrue(not torch.allclose(outputs.logits, outputs_lora.logits))
+        self.assertTrue(torch.allclose(outputs_lora.logits, outputs_reactivate.logits))
+
+    def test_swift_prompt_forward(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        prompt_config = PromptConfig(
+            dim=model.config.hidden_size, target_modules=r'.*layer\.\d+$', embedding_pos=0, attention_mask_pos=1)
+        outputs = model(**inputs)
+        model = Swift.prepare_model(model, config=prompt_config)
+        outputs_lora = model(**inputs)
+        model.deactivate_adapter('default')
+        outputs_deactivate = model(**inputs)
+        model.activate_adapter('default')
+        outputs_reactivate = model(**inputs)
+        self.assertTrue(torch.allclose(outputs.logits, outputs_deactivate.logits))
+        self.assertTrue(not torch.allclose(outputs.logits, outputs_lora.logits))
+        self.assertTrue(torch.allclose(outputs_lora.logits, outputs_reactivate.logits))
+
+    def test_swift_restuner_forward(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        restuner_config = ResTuningConfig(
+            dims=model.config.hidden_size,
+            root_modules=r'.*layer.0$',
+            stem_modules=r'.*layer\.\d+$',
+            target_modules=r'.*pooler',
+            target_modules_hook='input',
+            tuner_cfg='res_adapter',
+        )
+        outputs = model(**inputs)
+        model = Swift.prepare_model(model, config=restuner_config)
+        outputs_lora = model(**inputs)
+        model.deactivate_adapter('default')
+        outputs_deactivate = model(**inputs)
+        model.activate_adapter('default')
+        outputs_reactivate = model(**inputs)
+        self.assertTrue(torch.allclose(outputs.logits, outputs_deactivate.logits))
+        self.assertTrue(not torch.allclose(outputs.logits, outputs_lora.logits))
+        self.assertTrue(torch.allclose(outputs_lora.logits, outputs_reactivate.logits))
+
+    def lora_injection_with_dtype(self, dtype=torch.float32):
+        from swift.tuners.lora import Linear
+
+        def reset_lora_parameters(self, adapter_name, init_lora_weights):
+            if init_lora_weights is False:
+                return
+
+            if adapter_name in self.lora_A.keys():
+                if init_lora_weights is True:
+                    nn.init.kaiming_uniform_(self.lora_A[adapter_name].weight, a=math.sqrt(5))
+                elif init_lora_weights.lower() == 'gaussian':
+                    nn.init.normal_(self.lora_A[adapter_name].weight, std=1 / self.r[adapter_name])
+                else:
+                    raise ValueError(f'Unknown initialization {init_lora_weights=}')
+                nn.init.ones_(self.lora_B[adapter_name].weight)
+            if adapter_name in self.lora_embedding_A.keys():
+                # initialize a the same way as the default for nn.linear and b to zero
+                nn.init.ones_(self.lora_embedding_A[adapter_name])
+                nn.init.normal_(self.lora_embedding_B[adapter_name])
+
+        Linear.reset_lora_parameters = reset_lora_parameters
+
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        input = preprocessor('this is a test')
+        model = model.to(dtype)
+        model2 = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        model2 = model2.to(dtype)
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        model = Swift.prepare_model(model, config=lora_config)
+        self.assertTrue(isinstance(model, SwiftModel))
+        output1 = model(**input)
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default', WEIGHTS_NAME)))
+
+        model2 = Swift.from_pretrained(model2, self.tmp_dir, adapter_name={'default': 'test'})
+        self.assertTrue('test' in model2.adapters)
+        output2 = model2(**input)
+        self.assertTrue(torch.allclose(output1.logits, output2.logits))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir)
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+        if dtype == torch.float32 and os.environ.get('USE_UNIQUE_THREAD') == '1':
+            Swift.merge_and_unload(model2)
+            output3 = model2(**input)
+            self.assertTrue(torch.allclose(output1.logits, output3.logits))
+
+    def test_swift_lora_injection(self):
+        self.lora_injection_with_dtype()
+
+    def test_swift_lora_injection_bf16(self):
+        self.lora_injection_with_dtype(torch.bfloat16)
+
+    def test_save_to_peft_mix(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        adapter_config = AdapterConfig(
+            dim=model.config.hidden_size,
+            target_modules=r'.*layer\.\d+$',
+            method_name='feed_forward_chunk',
+            hidden_pos=0)
+        model = Swift.prepare_model(model, config={'lora': lora_config, 'adapter': adapter_config})
+        model.save_pretrained(os.path.join(self.tmp_dir, 'original'))
+        try:
+            Swift.save_to_peft_format(os.path.join(self.tmp_dir, 'original'), os.path.join(self.tmp_dir, 'converted'))
+            self.assertTrue(False)
+        except AssertionError as e:
+            print(e)
+            pass
+
+    def test_save_to_peft_param(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'], lora_dtype='float16')
+        model = Swift.prepare_model(model, config={'lora': lora_config})
+        model.save_pretrained(os.path.join(self.tmp_dir, 'original'))
+        try:
+            Swift.save_to_peft_format(os.path.join(self.tmp_dir, 'original'), os.path.join(self.tmp_dir, 'converted'))
+            self.assertTrue(False)
+        except AssertionError as e:
+            print(e)
+            pass
+
+    def test_save_to_peft_ok(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'], use_dora=True)
+        lora2_config = LoRAConfig(target_modules=['query', 'key', 'value'], use_dora=True)
+        model = Swift.prepare_model(model, config={'default': lora_config, 'lora': lora2_config})
+        model.save_pretrained(os.path.join(self.tmp_dir, 'original'))
+        Swift.save_to_peft_format(os.path.join(self.tmp_dir, 'original'), os.path.join(self.tmp_dir, 'converted'))
+        # A duplicate conversion
+        Swift.save_to_peft_format(os.path.join(self.tmp_dir, 'original'), os.path.join(self.tmp_dir, 'converted'))
+
+        # -------------------base case--------------------
+        model2 = SbertForSequenceClassification(SbertConfig())
+        model2 = PeftModel.from_pretrained(model2, os.path.join(self.tmp_dir, 'converted'))
+        model2.load_adapter(os.path.join(os.path.join(self.tmp_dir, 'converted'), 'lora'), 'lora')
+        state_dict = model.state_dict()
+        state_dict2 = {
+            key[len('base_model.model.'):]: value
+            for key, value in model2.state_dict().items() if 'lora' in key
+        }
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+        # -------------------override case--------------------
+        Swift.save_to_peft_format(os.path.join(self.tmp_dir, 'converted'), os.path.join(self.tmp_dir, 'converted'))
+        model2 = SbertForSequenceClassification(SbertConfig())
+        model2 = PeftModel.from_pretrained(model2, os.path.join(self.tmp_dir, 'converted'))
+        model2.load_adapter(os.path.join(os.path.join(self.tmp_dir, 'converted'), 'lora'), 'lora')
+        state_dict = model.state_dict()
+        state_dict2 = {
+            key[len('base_model.model.'):]: value
+            for key, value in model2.state_dict().items() if 'lora' in key
+        }
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+    def test_swift_multiple_adapters(self):
+        model = SbertForSequenceClassification(SbertConfig())
+        model2 = copy.deepcopy(model)
+        lora_config = LoRAConfig(target_modules=['query', 'key', 'value'])
+        adapter_config = AdapterConfig(
+            dim=model.config.hidden_size,
+            target_modules=r'.*layer\.\d+$',
+            method_name='feed_forward_chunk',
+            hidden_pos=0)
+        model = Swift.prepare_model(model, config={'lora': lora_config, 'adapter': adapter_config})
+        self.assertTrue(isinstance(model, SwiftModel))
+        model.save_pretrained(self.tmp_dir, adapter_name=['lora', 'adapter'])
+        with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f:
+            f.write('{}')
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'lora')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'lora', WEIGHTS_NAME)))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'adapter')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'adapter', WEIGHTS_NAME)))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir, adapter_name=['lora', 'adapter'])
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+    def test_part(self):
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        model = SbertForSequenceClassification.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        model_origin = copy.deepcopy(model)
+        model2 = copy.deepcopy(model)
+        targets = r'.*(query|key|value).*'
+        part_config = PartConfig(target_modules=targets)
+        model = Swift.prepare_model(model, config={'part': part_config})
+        self.assertTrue(isinstance(model, SwiftModel))
+
+        model.base_model.encoder.encoder.layer[0].attention.self.query._part_part.weight.data = torch.ones_like(
+            model.base_model.encoder.encoder.layer[0].attention.self.query._part_part.weight.data)
+
+        for name, module in model.named_modules():
+            if re.fullmatch(targets, name) and '_part_' not in name:
+                self.assertTrue(not module.weight.requires_grad)
+                self.assertTrue(model.get_submodule(name + '._part_part').weight.requires_grad)
+
+        model.save_pretrained(self.tmp_dir, adapter_name=['part'])
+        with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f:
+            f.write('{}')
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'part')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'part', WEIGHTS_NAME)))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir, adapter_name=['part'])
+        self.assertTrue(
+            all(
+                torch.isclose(model.base_model.encoder.encoder.layer[0].attention.self.query._part_part.weight.data,
+                              model2.base_model.encoder.encoder.layer[0].attention.self.query._part_part.weight.data).
+                flatten().detach().cpu()))
+
+        state_dict = model.model.state_dict()
+        state_dict2 = model2.model.state_dict()
+        self.assertTrue(str(state_dict) == str(state_dict2))
+
+        output = model(**inputs)
+        output2 = model2(**inputs)
+        output_origin = model_origin(**inputs)
+        self.assertTrue(all(torch.isclose(output.logits, output2.logits).flatten().detach().cpu()))
+        self.assertTrue(not all(torch.isclose(output_origin.logits, output2.logits).flatten().detach().cpu()))
+
+        model2.deactivate_adapter('part')
+        output = model(**inputs)
+        output2 = model2(**inputs)
+        output_origin = model_origin(**inputs)
+        self.assertTrue(not all(torch.isclose(output.logits, output2.logits).flatten().detach().cpu()))
+        self.assertTrue(all(torch.isclose(output_origin.logits, output2.logits).flatten().detach().cpu()))
+
+        model2.activate_adapter('part')
+        output = model(**inputs)
+        output2 = model2(**inputs)
+        output_origin = model_origin(**inputs)
+        self.assertTrue(all(torch.isclose(output.logits, output2.logits).flatten().detach().cpu()))
+        self.assertTrue(not all(torch.isclose(output_origin.logits, output2.logits).flatten().detach().cpu()))
+
+        targets = r'.*(query|key|value).*'
+        part_config = PartConfig(target_modules=targets)
+        lora_config = LoRAConfig(target_modules=targets)
+        model2 = Swift.prepare_model(model2, config={'part2': part_config})
+        model2 = Swift.prepare_model(model2, config={'lora': lora_config})
+        model2 = Swift.prepare_model(model2, config={'part3': part_config})
+        model2.set_active_adapters('part2', offload='meta')
+        model2.set_active_adapters('part3', offload='meta')
+        model2.set_active_adapters('lora', offload='meta')
+        model2.set_active_adapters('part2', offload='meta')
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part.activated)
+        self.assertTrue(
+            model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part2.activated)
+        model2.set_active_adapters('part', offload='meta')
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part2.activated)
+        self.assertTrue(model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part.activated)
+        output = model(**inputs)
+        output2 = model2(**inputs)
+        output_origin = model_origin(**inputs)
+        self.assertTrue(all(torch.isclose(output.logits, output2.logits).flatten().detach().cpu()))
+        self.assertTrue(not all(torch.isclose(output_origin.logits, output2.logits).flatten().detach().cpu()))
+
+        model2.set_active_adapters('part2', offload='meta')
+        model2.deactivate_adapter('part2', offload='meta')
+        model2.deactivate_adapter('lora', offload='cpu')
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part2.activated)
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part.activated)
+        output = model(**inputs)
+        output2 = model2(**inputs)
+        output_origin = model_origin(**inputs)
+        self.assertTrue(not all(torch.isclose(output.logits, output2.logits).flatten().detach().cpu()))
+        self.assertTrue(all(torch.isclose(output_origin.logits, output2.logits).flatten().detach().cpu()))
+        model2.activate_adapter('lora')
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part2.activated)
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part.activated)
+        self.assertTrue(
+            not model2.base_model.encoder.encoder.layer[0].attention.self.query.base_layer._part_part3.activated)
+        self.assertTrue(model2.base_model.encoder.encoder.layer[0].attention.self.query.active_adapters == ['lora'])
+
+    def test_swift_multiple_adapters_switching(self):
+        from swift.tuners.lora import Linear
+        from swift.tuners.adapter import AdapterModule
+
+        def reset_lora_parameters(self, adapter_name, init_lora_weights):
+            if init_lora_weights is False:
+                return
+
+            if adapter_name in self.lora_A.keys():
+                if init_lora_weights is True:
+                    # initialize A the same way as the default for nn.Linear and B to zero
+                    # https://github.com/microsoft/LoRA/blob/a0a92e0f26c067cf94747bdbf1ce73793fa44d19/loralib/layers.py#L124
+                    nn.init.ones_(self.lora_A[adapter_name].weight)
+                elif init_lora_weights.lower() == 'gaussian':
+                    nn.init.normal_(self.lora_A[adapter_name].weight, std=1 / self.r[adapter_name])
+                else:
+                    raise ValueError(f'Unknown initialization {init_lora_weights=}')
+                nn.init.ones_(self.lora_B[adapter_name].weight)
+            if adapter_name in self.lora_embedding_A.keys():
+                # initialize a the same way as the default for nn.linear and b to zero
+                nn.init.ones_(self.lora_embedding_A[adapter_name])
+                nn.init.normal_(self.lora_embedding_B[adapter_name])
+
+        Linear.reset_lora_parameters = reset_lora_parameters
+
+        def init_weights(self):
+
+            def _init_weights(m):
+                if isinstance(m, nn.Linear):
+                    nn.init.ones_(m.weight)
+                    nn.init.ones_(m.bias)
+
+            self.apply(_init_weights)
+
+        AdapterModule.init_weights = init_weights
+
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        model1 = copy.deepcopy(model)
+        model2 = copy.deepcopy(model)
+        model1 = Swift.prepare_model(
+            model1,
+            config={
+                'lora1':
+                LoRAConfig(target_modules=['query', 'key', 'value']),
+                'adapter1':
+                AdapterConfig(
+                    dim=model.config.hidden_size,
+                    target_modules=r'.*layer\.\d+$',
+                    method_name='feed_forward_chunk',
+                    hidden_pos=0)
+            })
+        model2 = Swift.prepare_model(
+            model2,
+            config={
+                'lora2':
+                LoRAConfig(target_modules=['query', 'key', 'value']),
+                'adapter2':
+                AdapterConfig(
+                    dim=model.config.hidden_size,
+                    target_modules=r'.*layer\.\d+$',
+                    method_name='feed_forward_chunk',
+                    hidden_pos=0)
+            })
+        model = Swift.prepare_model(
+            model,
+            config={
+                'lora1': LoRAConfig(target_modules=['query', 'key', 'value']),
+                'lora2': LoRAConfig(target_modules=['query', 'key', 'value']),
+            })
+
+        model = Swift.prepare_model(
+            model,
+            config={
+                'adapter1':
+                AdapterConfig(
+                    dim=model.config.hidden_size,
+                    target_modules=r'.*layer\.\d+$',
+                    method_name='feed_forward_chunk',
+                    hidden_pos=0),
+                'adapter2':
+                AdapterConfig(
+                    dim=model.config.hidden_size,
+                    target_modules=r'.*layer\.\d+$',
+                    method_name='feed_forward_chunk',
+                    hidden_pos=0),
+            })
+
+        model.deactivate_adapter('adapter2', offload='meta')
+        model.deactivate_adapter('lora2', offload='meta')
+        outputs1 = model(**inputs)
+        outputs2 = model1(**inputs)
+        self.assertTrue(torch.allclose(outputs1.logits, outputs2.logits))
+        model.activate_adapter('adapter2')
+        model.activate_adapter('lora2')
+        model.deactivate_adapter('adapter1', offload='meta')
+        model.deactivate_adapter('lora1', offload='meta')
+        outputs1 = model(**inputs)
+        outputs2 = model2(**inputs)
+        self.assertTrue(torch.allclose(outputs1.logits, outputs2.logits))
+
+        if os.environ.get('USE_UNIQUE_THREAD') == '0':
+
+            def thread_func1():
+                model1.set_active_adapters(['lora1', 'adapter1'], offload=None)
+                model.set_active_adapters(['lora1', 'adapter1'], offload=None)
+                outputs_single = model1(**inputs)
+                outputs_t1 = model(**inputs)
+                self.assertTrue(torch.allclose(outputs_single.logits, outputs_t1.logits))
+
+            def thread_func2():
+                model2.set_active_adapters(['lora2', 'adapter2'], offload=None)
+                model.set_active_adapters(['lora2', 'adapter2'], offload=None)
+                outputs_single = model2(**inputs)
+                outputs_t2 = model(**inputs)
+                self.assertTrue(torch.allclose(outputs_single.logits, outputs_t2.logits))
+
+            with ThreadPoolExecutor(2) as executor:
+                f1 = executor.submit(thread_func1)
+                f2 = executor.submit(thread_func2)
+                e1 = f1.exception()
+                e2 = f2.exception()
+                if e1 is not None:
+                    raise e1
+                if e2 is not None:
+                    raise e2
+
+    def test_swift_side_bert(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        inputs = preprocessor('how are you')
+        model2 = copy.deepcopy(model)
+        result_origin = model(**inputs).logits
+        print(f'test_swift_side_bert result_origin shape: {result_origin.shape}, '
+              f'result_origin sum: {torch.sum(result_origin)}')
+
+        side_config = SideConfig(
+            dim=model.config.hidden_size,
+            target_modules=r'.*encoder.encoder',
+            side_module_name='mlp',
+            target_hidden_pos='last_hidden_state')
+
+        model = Swift.prepare_model(model, config=side_config)
+        result_activate = model(**inputs).logits
+        model.deactivate_adapter('default')
+        result_deactivate = model(**inputs).logits
+        model.activate_adapter('default')
+        result_reactivate = model(**inputs).logits
+        self.assertTrue(torch.allclose(result_origin, result_deactivate))
+        self.assertTrue(not torch.allclose(result_origin, result_activate))
+        self.assertTrue(torch.allclose(result_activate, result_reactivate))
+        print(f'test_swift_side_bert result shape: {result_origin.shape}, result sum: {torch.sum(result_origin)}')
+
+        self.assertTrue(isinstance(model, SwiftModel))
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default', WEIGHTS_NAME)))
+
+        model2 = Swift.from_pretrained(model2, self.tmp_dir)
+
+        state_dict = model.state_dict()
+        state_dict2 = model2.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/tuners/test_swift_device_map.py

@@ -0,0 +1,45 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+import torch
+from modelscope import Model
+from peft.utils import WEIGHTS_NAME
+
+from swift import LoRAConfig, SwiftModel
+
+
+@unittest.skip
+class TestSwift(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def test_swift_multiple_adapters(self):
+        model = Model.from_pretrained('modelscope/Llama-2-7b-ms', device_map='auto')
+        lora_config = LoRAConfig(target_modules=['q_proj', 'k_proj', 'v_proj'])
+        model: SwiftModel = SwiftModel(model, config={'lora': lora_config})
+        self.assertTrue(isinstance(model, SwiftModel))
+        model.save_pretrained(self.tmp_dir, adapter_name=['lora'])
+        state_dict = model.state_dict()
+        with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f:
+            f.write('{}')
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'lora')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'lora', WEIGHTS_NAME)))
+        model = Model.from_pretrained('modelscope/Llama-2-7b-ms', device_map='auto')
+        model = SwiftModel.from_pretrained(model, self.tmp_dir, adapter_name=['lora'], device_map='auto')
+
+        state_dict2 = model.state_dict()
+        for key in state_dict:
+            self.assertTrue(key in state_dict2)
+            self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu()))
+
+        self.assertTrue(len(set(model.hf_device_map.values())) == torch.cuda.device_count())

tests/tuners/test_swift_restuning.py

@@ -0,0 +1,136 @@
+import copy
+import os
+import shutil
+import tempfile
+import unittest
+
+import torch
+from modelscope import snapshot_download
+
+from swift import ResTuningConfig, Swift, SwiftModel
+
+
+class TestSwiftResTuning(unittest.TestCase):
+
+    def setUp(self):
+        print(('Testing %s.%s' % (type(self).__name__, self._testMethodName)))
+        self.tmp_dir = tempfile.TemporaryDirectory().name
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+        super().tearDown()
+
+    def set_random_seed(self, seed=123):
+        """Set random seed manually to get deterministic results"""
+        import random
+        import numpy as np
+        import torch
+        random.seed(seed)
+        np.random.seed(seed)
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed(seed)
+        torch.cuda.manual_seed_all(seed)
+
+    def model_comparison(self, model, model2):
+        model_key = list(model.state_dict().keys())
+        model2_key = list(model2.state_dict().keys())
+        self.assertTrue(model_key == model2_key)
+        model_val = torch.sum(torch.stack([torch.sum(val) for val in model.state_dict().values()]))
+        model2_val = torch.sum(torch.stack([torch.sum(val) for val in model2.state_dict().values()]))
+        self.assertTrue(torch.isclose(model_val, model2_val))
+
+    def test_swift_restuning_vit(self):
+        model_dir = snapshot_download('AI-ModelScope/vit-base-patch16-224')
+        from transformers import AutoModelForImageClassification
+        model = AutoModelForImageClassification.from_pretrained(model_dir)
+        model_swift_1 = copy.deepcopy(model)
+        model_swift_2 = copy.deepcopy(model)
+        result_origin = model(torch.ones((1, 3, 224, 224))).logits
+        print(f'test_swift_restuning_vit result_origin shape: {result_origin.shape}, '
+              f'result_origin sum: {torch.sum(result_origin)}')
+
+        # load type - 1
+        self.set_random_seed()
+        restuning_config_1 = ResTuningConfig(
+            dims=768,
+            root_modules=r'.*vit.encoder.layer.0$',
+            stem_modules=r'.*vit.encoder.layer\.\d+$',
+            target_modules=r'.*vit.layernorm',
+            target_modules_hook='input',
+            tuner_cfg='res_adapter',
+        )
+        model_swift_1 = Swift.prepare_model(model_swift_1, config=restuning_config_1)
+        self.assertTrue(isinstance(model_swift_1, SwiftModel))
+        print(model_swift_1.get_trainable_parameters())
+        result_swift_1 = model_swift_1(torch.ones((1, 3, 224, 224))).logits
+        print(f'test_swift_restuning_vit result_swift_1 shape: {result_swift_1.shape}, '
+              f'result_swift_1 sum: {torch.sum(result_swift_1)}')
+
+        # load type - 2
+        self.set_random_seed()
+        restuning_config_2 = ResTuningConfig(
+            dims=768,
+            root_modules=r'.*vit.encoder.layer.0$',
+            stem_modules=r'.*vit.encoder.layer\.\d+$',
+            target_modules=r'.*vit.encoder',
+            target_modules_hook='output',
+            target_hidden_pos='last_hidden_state',
+            tuner_cfg='res_adapter',
+        )
+        model_swift_2 = Swift.prepare_model(model_swift_2, config=restuning_config_2)
+        self.assertTrue(isinstance(model_swift_2, SwiftModel))
+        print(model_swift_2.get_trainable_parameters())
+        result_swift_2 = model_swift_2(torch.ones((1, 3, 224, 224))).logits
+        print(f'test_swift_restuning_vit result_swift_2 shape: {result_swift_2.shape}, '
+              f'result_swift_2 sum: {torch.sum(result_swift_2)}')
+
+        self.assertTrue(all(torch.isclose(result_swift_1, result_swift_2).flatten()))
+
+        model_swift_1.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        model_loaded = Swift.from_pretrained(model, self.tmp_dir)
+        self.model_comparison(model_swift_1, model_loaded)
+
+    def test_swift_restuning_diffusers_sd(self):
+        model_dir = snapshot_download('AI-ModelScope/stable-diffusion-v1-5')
+        from diffusers import UNet2DConditionModel
+        model = UNet2DConditionModel.from_pretrained(model_dir, subfolder='unet')
+        model.requires_grad_(False)
+        model2 = copy.deepcopy(model)
+        self.set_random_seed()
+        input_data = {
+            'sample': torch.ones((1, 4, 64, 64)),
+            'timestep': 10,
+            'encoder_hidden_states': torch.ones((1, 77, 768))
+        }
+        result_origin = model(**input_data).sample
+        print(f'test_swift_restuning_diffusers_sd result_origin shape: {result_origin.shape}, '
+              f'result_origin sum: {torch.sum(result_origin)}')
+
+        self.set_random_seed()
+        restuning_config = ResTuningConfig(
+            dims=[1280, 1280, 1280, 640, 320],
+            root_modules='mid_block',
+            stem_modules=['mid_block', 'up_blocks.0', 'up_blocks.1', 'up_blocks.2', 'up_blocks.3'],
+            target_modules='conv_norm_out',
+            tuner_cfg='res_group_adapter',
+            use_upsample=True,
+            upsample_out_channels=[1280, 1280, 640, 320, None],
+            zero_init_last=True)
+
+        model = Swift.prepare_model(model, config=restuning_config)
+        self.assertTrue(isinstance(model, SwiftModel))
+        print(model.get_trainable_parameters())
+
+        result = model(**input_data).sample
+        print(f'test_swift_restuning_diffusers_sd result shape: {result.shape}, result sum: {torch.sum(result)}')
+        model.save_pretrained(self.tmp_dir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'default')))
+        model2 = Swift.from_pretrained(model2, self.tmp_dir)
+        self.model_comparison(model, model2)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/utils/test_file_utils.py

@@ -0,0 +1,32 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+from swift.utils import copy_files_by_pattern
+
+
+class TestFileUtils(unittest.TestCase):
+
+    def setUp(self):
+        self._tmp_dir = tempfile.TemporaryDirectory()
+        self.tmp_dir = self._tmp_dir.name
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_copy_files(self):
+        os.makedirs(os.path.join(self.tmp_dir, 'source'))
+        os.makedirs(os.path.join(self.tmp_dir, 'source', 'subfolder'))
+        with open(os.path.join(self.tmp_dir, 'source', '1.txt'), 'w') as f:
+            f.write('')
+        with open(os.path.join(self.tmp_dir, 'source', 'subfolder', '2.txt'), 'w') as f:
+            f.write('')
+        copy_files_by_pattern(
+            os.path.join(self.tmp_dir, 'source'), os.path.join(self.tmp_dir, 'target'), ['*.txt', 'subfolder/*.txt'])
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'target', '1.txt')))
+        self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'target', 'subfolder', '2.txt')))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/utils/test_io_utils.py

@@ -0,0 +1,42 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+from swift.utils import append_to_jsonl, get_logger, read_from_jsonl, write_to_jsonl
+
+logger = get_logger()
+
+
+class TestIOUtils(unittest.TestCase):
+
+    def setUp(self):
+        self._tmp_dir = tempfile.TemporaryDirectory()
+        self.tmp_dir = self._tmp_dir.name
+        # self.tmp_dir = 'test'
+        logger.info(f'self.tmp_dir: {self.tmp_dir}')
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_jsonl(self):
+        fpath = os.path.join(self.tmp_dir, '1.jsonl')
+        obj_list = [{'aaa': 'bbb'}, 111, [1.1]]
+        write_to_jsonl(fpath, obj_list)
+        new_obj = {'bbb': 'aaa'}
+        obj_list.append(new_obj)
+        append_to_jsonl(fpath, new_obj)
+        new_obj_list = read_from_jsonl(fpath)
+        self.assertTrue(new_obj_list == obj_list)
+
+    def test_jsonl2(self):
+        fpath = os.path.join(self.tmp_dir, '1.jsonl')
+        obj_list = [{'aaa': 'bbb'}, 111, [1.1]]
+        for obj in obj_list:
+            append_to_jsonl(fpath, obj)
+        new_obj_list = read_from_jsonl(fpath)
+        self.assertTrue(new_obj_list == obj_list)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/utils/test_split_str_parts_by.py

@@ -0,0 +1,13 @@
+from swift.llm.template import split_str_parts_by
+
+
+def test_split_str_parts_by():
+    print(split_str_parts_by('aaaAction:bb\nbAction Inputs:\nabbb', ['Action:', 'Action Inputs:'], regex_mode=False))
+    print(split_str_parts_by('aaaAction:bb\nbAction Inputs:\nabbb', ['Action:', 'Action Inputs:'], regex_mode=True))
+    print(split_str_parts_by('aaa<tool_call>bbb</tool_call>ccc', ['<tool_call>.+?</tool_call>'], regex_mode=True))
+    print(split_str_parts_by('aaa<image>\nbb\nb<audio>\nabbb', ['<image>', '<audio>', '<video>'], regex_mode=False))
+    print(split_str_parts_by('aaa<image>\nbb\nb<audio>\nabbb', ['<image>', '<audio>', '<video>'], regex_mode=True))
+
+
+if __name__ == '__main__':
+    test_split_str_parts_by()

tests/utils/test_torch_utils.py

@@ -0,0 +1,16 @@
+import unittest
+
+from modelscope import Model
+
+from swift.utils.torch_utils import find_sub_module
+
+
+class TestTorchUtils(unittest.TestCase):
+
+    def test_find_sub_module(self):
+        model = Model.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base')
+        self.assertTrue(find_sub_module(model, 'query') is not None)
+
+
+if __name__ == '__main__':
+    unittest.main()