Create datasets_loader.py

datasets_loader.py

@@ -0,0 +1,33 @@
+# datasets_loader.py
+
+from datasets import load_dataset, DatasetDict
+from transformers import AutoTokenizer
+
+def load_threat_dataset(path: str, tokenizer_name="bert-base-chinese"):
+    """
+    Loads a dataset of Chinese cybercrime posts with labels.
+    Expects columns: text, label
+    """
+    raw = load_dataset("csv", data_files=path)
+
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+
+    def tokenize(batch):
+        return tokenizer(
+            batch["text"],
+            truncation=True,
+            padding="max_length",
+            max_length=256
+        )
+
+    tokenized = raw.map(tokenize, batched=True)
+
+    return DatasetDict({
+        "train": tokenized["train"],
+        "test": tokenized["test"]
+    })
+
+
+# Example:
+# ds = load_threat_dataset("dataset/threat_samples.csv")
+# print(ds["train"][0])