Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+train_data/de_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/dna_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/en_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/es_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/fr_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/gene_eng.txt filter=lfs diff=lfs merge=lfs -text
+train_data/gene_eng_zh.txt filter=lfs diff=lfs merge=lfs -text
+train_data/gene_eng_zh_de_es.txt filter=lfs diff=lfs merge=lfs -text
+train_data/ja_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/ko_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/protein_4g.txt filter=lfs diff=lfs merge=lfs -text
+train_data/zh_wiki_4g.txt filter=lfs diff=lfs merge=lfs -text

get_data/.ipynb_checkpoints/get_wiki1-checkpoint.ipynb

@@ -0,0 +1,1422 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "163a0ec0-2003-40f8-bf8a-63c1146f9130",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"\\nimport os\\n\\n# 设置环境变量\\nos.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\\n\\n# 打印环境变量以确认设置成功\\nprint(os.environ.get('HF_ENDPOINT'))\\n\""
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value\n",
+    "\n",
+    "\"\"\"\n",
+    "import os\n",
+    "\n",
+    "# 设置环境变量\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
+    "\n",
+    "# 打印环境变量以确认设置成功\n",
+    "print(os.environ.get('HF_ENDPOINT'))\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "4aa15036-4be6-44ac-be6d-83389ae07c5d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from datasets import load_dataset\n",
+    "\n",
+    "# # 加载特定语言的 Wikipedia 语料\n",
+    "# langs = [\"20220301.en\", \"20220301.fr\", \"20220301.de\", \"20220301.zh\", \"20220301.ja\"]\n",
+    "# datasets = {lang: load_dataset(\"wikimedia/wikipedia\", lang, split=\"train\") for lang in langs}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "02351414-fcad-44ba-964c-8ced0a88e609",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "408855cc406d47a3a47803d00829febd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading dataset shards:   0%|          | 0/41 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 6458670\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "datasets_en = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.en\")\n",
+    "datasets_en"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f00a2e59-0aed-446e-9c7b-d21233bcdd1c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"en_wiki.txt\",\"w\")\n",
+    "for item in datasets_en[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ce2039da-ae06-49de-87da-bc452ee1f252",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d240fe1bef53424e9ccbb8af29346f0c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00015.parquet:   0%|          | 0.00/764M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "807912e6be304fcaa0786f2c863fa8ab",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00015.parquet:   0%|          | 0.00/416M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ff2349e16ea44a1a9033f04096cf0fe4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00015.parquet:   0%|          | 0.00/342M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "428705e16387469abc2f70d8f01032c8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00015.parquet:   0%|          | 0.00/306M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3edadedc0767441fa441450e541d0a41",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00015.parquet:   0%|          | 0.00/281M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "485c9747448e4fc792c748e3920d893c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00015.parquet:   0%|          | 0.00/261M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "32acc84b51494535aff3fe9edf17d0f9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00015.parquet:   0%|          | 0.00/220M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1626f0e6390542298f9c22e659e95d52",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00015.parquet:   0%|          | 0.00/210M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5c9d6d941a564add86f644daa4710850",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00015.parquet:   0%|          | 0.00/215M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "13b0fe00bca644d9b971793375c2bc3b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00015.parquet:   0%|          | 0.00/211M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "60ff775f33b74bf6b9be369f4b38e116",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00015.parquet:   0%|          | 0.00/181M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "862390a8eff241b3b371e6db7404591a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00015.parquet:   0%|          | 0.00/197M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6162884ffd804cffb1f0d1e6a57e940f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00015.parquet:   0%|          | 0.00/180M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0d8e1f181e4043019e43704b685851f2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00013-of-00015.parquet:   0%|          | 0.00/219M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "fc7550b48dbe4d35b1c5e1c7ff52f0a8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00014-of-00015.parquet:   0%|          | 0.00/220M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1d84178aa296430cb2e949c37fbde0e2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/2402095 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 2402095\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.fr\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ac610da1-9529-491a-a9f4-5d6f3a92cce7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"fr_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8be4a828-affd-4b07-8821-16cbefb93bd6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2a4a8425-8615-45c5-84cf-fcfb6a54727a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1ade008c4100486e9e47a52fadc1ece7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data:   0%|          | 0/18 [00:00<?, ?files/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bbe51753a90240d399d26dfd0bdb1aa6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00018.parquet:   0%|          | 0.00/239M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "717f471eccca49e5a415f8b8b8c9c6e3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00018.parquet:   0%|          | 0.00/252M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1ea2609792144411afa2a855e2ff1c7d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00018.parquet:   0%|          | 0.00/283M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a8c88a13fd5d475085649381fe2fbba7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00018.parquet:   0%|          | 0.00/241M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0b63f5caf6c54225b35a3bdc915cc73e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00018.parquet:   0%|          | 0.00/773M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "de0f6e9973ab427eaa937924b1134029",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00018.parquet:   0%|          | 0.00/446M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "beac8579da8244deadc2da23aec6ec6e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00015-of-00018.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "293b420636b44bab986752b92c91a1b3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00018.parquet:   0%|          | 0.00/236M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d3d66b5d72ac472aa3ba20dae34f9695",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00018.parquet:   0%|          | 0.00/245M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3be8b392a6614472b005ba9b9b1e1d59",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00018.parquet:   0%|          | 0.00/277M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b371f42b410e4cbf9137666b4bac2647",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00018.parquet:   0%|          | 0.00/256M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a9ab370c0f534b289fdac4ed3cddff3a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00018.parquet:   0%|          | 0.00/369M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dce437fe46e04c82b2f8ab7a3121592b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00018.parquet:   0%|          | 0.00/291M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5bec466db34b4daa926a6eef48273817",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00014-of-00018.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e47f796b69749ef8f3bf255300553ec",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00018.parquet:   0%|          | 0.00/300M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b6dd9bf441d24ffa886996c210b4607f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00013-of-00018.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0fa5351d1be542b88a3024b900b0ef19",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00016-of-00018.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1452efc3d5564ef99ae336dbcc36b22f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00017-of-00018.parquet:   0%|          | 0.00/222M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c12fa5a2d21241c6a31b4c889999ad0a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/2665357 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ca6b84a24ecf4e00bf8e550ba5eb5054",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading dataset shards:   0%|          | 0/18 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 2665357\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.de\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1c60e53d-3031-44e7-b1da-edc419079f13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"de_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c42100db-79d9-4cbe-9582-e43d58567173",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "eee3698d-ccb8-4cec-9796-e93886fd6b1e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a03b4c97410645eaaae45f25e1c03108",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00015.parquet:   0%|          | 0.00/612M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5a03f25cd8354e3299b82554c0fc3287",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00015.parquet:   0%|          | 0.00/383M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8a63110f3cf648eda442ee3177aee2f6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00015.parquet:   0%|          | 0.00/305M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "09ec9ce700084abcb944f6b6669ebf15",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00015.parquet:   0%|          | 0.00/252M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "363a3b6272c24e23a0e708864c872310",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00015.parquet:   0%|          | 0.00/240M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "58b7f596d96246439e7cc21acc333fb4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00015.parquet:   0%|          | 0.00/240M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2872cc14f4944b4894bd05bbb256d892",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00015.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3d65c3b76a02445e85d6723ea19565c2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00015.parquet:   0%|          | 0.00/218M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a77dd5822bd34896b4dbd4d22cb8331c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00015.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "52ce12211c8743acac275af2c7ac049c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00015.parquet:   0%|          | 0.00/235M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9da3788934f14b15a63e4add38e989b4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00015.parquet:   0%|          | 0.00/217M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "06254bec73a24830b685468235260f0c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00015.parquet:   0%|          | 0.00/213M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d25a6d36b72446458525dd060d63102a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00015.parquet:   0%|          | 0.00/201M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4da023af77894a72a1e6223c91820983",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00013-of-00015.parquet:   0%|          | 0.00/193M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6c6d63c3e4df46c69da6b42c8b7eae4a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00014-of-00015.parquet:   0%|          | 0.00/176M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f98c43bad8a140cca38d7c00232f884a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/1389467 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 1389467\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.ja\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "4b61a5b0-85ca-4a0e-99b0-0ea2e3bc9e82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"ja_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5c56f7fd-8188-4d30-928d-e307764f23f9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "1668c12c-cbfe-4ea8-82c6-4e27607e9b67",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8f8cd7bb3992440c89087f809b96832d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00013.parquet:   0%|          | 0.00/688M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b6c6f856ff3f4070a26e56bfacca6deb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00013.parquet:   0%|          | 0.00/376M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d90dc9bb27ef418e906109adf3e7fad3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00013.parquet:   0%|          | 0.00/287M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e72006a2edd24c3a9bd88a0e2ac57d7b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00013.parquet:   0%|          | 0.00/245M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "df667cedb5374cd39e86e52c6cea3a7a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00013.parquet:   0%|          | 0.00/168M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6b005307539743769e0ac974c2996d9b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00013.parquet:   0%|          | 0.00/178M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c749d7a330f24d2fa684857be0700cc3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00013.parquet:   0%|          | 0.00/216M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7b07061ea4694ba8af395eeae31bd1fd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00013.parquet:   0%|          | 0.00/241M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e07a225a3f24d9a9c92ad9ee153b066",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00013.parquet:   0%|          | 0.00/227M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3054fd1470a34bbab1c6dee28dcb2441",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00013.parquet:   0%|          | 0.00/223M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "90f7a09fb95b4c4a926711a258c32107",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00013.parquet:   0%|          | 0.00/167M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f2ceb836cfca47e2ba75659dacfc9739",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00013.parquet:   0%|          | 0.00/254M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b20a7cea4fb146a5810cde198ef60ab6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00013.parquet:   0%|          | 0.00/226M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ba34e3fba4be480cbf145281eb834d64",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/1841155 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 1841155\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.es\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "9b3d2f1d-47c3-4a4d-8f60-cf5874ec643b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"es_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d300900-2d36-4f21-a799-17b466239666",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "1d20d238-3015-443f-956a-ca1a7c622001",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 1384748\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.zh\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "eac16b65-44d1-43ae-9aeb-c06a470daf0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"zh_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67c13fd3-ce11-41ce-8ca9-7d1d783310fb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "8d4f223d-accc-44f5-acf3-5eca7621e960",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2b39607ab45b4449b96d05858de65c90",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00003.parquet:   0%|          | 0.00/400M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c930ca1ff58a4ba2995158c1944335c2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00003.parquet:   0%|          | 0.00/205M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "37654989816342af9a355a2cb6e770b4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00003.parquet:   0%|          | 0.00/177M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b0dbd13bc9dc457eae92438fd63947ae",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/647897 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 647897\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.ko\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "1af1e848-559a-4882-a3ce-966a63edac44",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"ko_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4b3810a-f564-4a4e-8ccc-5f23f8541414",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

get_data/get_wiki1.ipynb

@@ -0,0 +1,1422 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "163a0ec0-2003-40f8-bf8a-63c1146f9130",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"\\nimport os\\n\\n# 设置环境变量\\nos.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\\n\\n# 打印环境变量以确认设置成功\\nprint(os.environ.get('HF_ENDPOINT'))\\n\""
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "\n",
+    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
+    "output = result.stdout\n",
+    "for line in output.splitlines():\n",
+    "    if '=' in line:\n",
+    "        var, value = line.split('=', 1)\n",
+    "        os.environ[var] = value\n",
+    "\n",
+    "\"\"\"\n",
+    "import os\n",
+    "\n",
+    "# 设置环境变量\n",
+    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
+    "\n",
+    "# 打印环境变量以确认设置成功\n",
+    "print(os.environ.get('HF_ENDPOINT'))\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "4aa15036-4be6-44ac-be6d-83389ae07c5d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from datasets import load_dataset\n",
+    "\n",
+    "# # 加载特定语言的 Wikipedia 语料\n",
+    "# langs = [\"20220301.en\", \"20220301.fr\", \"20220301.de\", \"20220301.zh\", \"20220301.ja\"]\n",
+    "# datasets = {lang: load_dataset(\"wikimedia/wikipedia\", lang, split=\"train\") for lang in langs}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "02351414-fcad-44ba-964c-8ced0a88e609",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "408855cc406d47a3a47803d00829febd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading dataset shards:   0%|          | 0/41 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 6458670\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "datasets_en = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.en\")\n",
+    "datasets_en"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f00a2e59-0aed-446e-9c7b-d21233bcdd1c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"en_wiki.txt\",\"w\")\n",
+    "for item in datasets_en[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ce2039da-ae06-49de-87da-bc452ee1f252",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d240fe1bef53424e9ccbb8af29346f0c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00015.parquet:   0%|          | 0.00/764M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "807912e6be304fcaa0786f2c863fa8ab",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00015.parquet:   0%|          | 0.00/416M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ff2349e16ea44a1a9033f04096cf0fe4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00015.parquet:   0%|          | 0.00/342M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "428705e16387469abc2f70d8f01032c8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00015.parquet:   0%|          | 0.00/306M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3edadedc0767441fa441450e541d0a41",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00015.parquet:   0%|          | 0.00/281M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "485c9747448e4fc792c748e3920d893c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00015.parquet:   0%|          | 0.00/261M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "32acc84b51494535aff3fe9edf17d0f9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00015.parquet:   0%|          | 0.00/220M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1626f0e6390542298f9c22e659e95d52",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00015.parquet:   0%|          | 0.00/210M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5c9d6d941a564add86f644daa4710850",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00015.parquet:   0%|          | 0.00/215M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "13b0fe00bca644d9b971793375c2bc3b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00015.parquet:   0%|          | 0.00/211M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "60ff775f33b74bf6b9be369f4b38e116",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00015.parquet:   0%|          | 0.00/181M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "862390a8eff241b3b371e6db7404591a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00015.parquet:   0%|          | 0.00/197M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6162884ffd804cffb1f0d1e6a57e940f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00015.parquet:   0%|          | 0.00/180M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0d8e1f181e4043019e43704b685851f2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00013-of-00015.parquet:   0%|          | 0.00/219M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "fc7550b48dbe4d35b1c5e1c7ff52f0a8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00014-of-00015.parquet:   0%|          | 0.00/220M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1d84178aa296430cb2e949c37fbde0e2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/2402095 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 2402095\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.fr\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ac610da1-9529-491a-a9f4-5d6f3a92cce7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"fr_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8be4a828-affd-4b07-8821-16cbefb93bd6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2a4a8425-8615-45c5-84cf-fcfb6a54727a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1ade008c4100486e9e47a52fadc1ece7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data:   0%|          | 0/18 [00:00<?, ?files/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bbe51753a90240d399d26dfd0bdb1aa6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00018.parquet:   0%|          | 0.00/239M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "717f471eccca49e5a415f8b8b8c9c6e3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00018.parquet:   0%|          | 0.00/252M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1ea2609792144411afa2a855e2ff1c7d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00018.parquet:   0%|          | 0.00/283M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a8c88a13fd5d475085649381fe2fbba7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00018.parquet:   0%|          | 0.00/241M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0b63f5caf6c54225b35a3bdc915cc73e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00018.parquet:   0%|          | 0.00/773M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "de0f6e9973ab427eaa937924b1134029",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00018.parquet:   0%|          | 0.00/446M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "beac8579da8244deadc2da23aec6ec6e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00015-of-00018.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "293b420636b44bab986752b92c91a1b3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00018.parquet:   0%|          | 0.00/236M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d3d66b5d72ac472aa3ba20dae34f9695",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00018.parquet:   0%|          | 0.00/245M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3be8b392a6614472b005ba9b9b1e1d59",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00018.parquet:   0%|          | 0.00/277M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b371f42b410e4cbf9137666b4bac2647",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00018.parquet:   0%|          | 0.00/256M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a9ab370c0f534b289fdac4ed3cddff3a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00018.parquet:   0%|          | 0.00/369M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dce437fe46e04c82b2f8ab7a3121592b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00018.parquet:   0%|          | 0.00/291M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5bec466db34b4daa926a6eef48273817",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00014-of-00018.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e47f796b69749ef8f3bf255300553ec",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00018.parquet:   0%|          | 0.00/300M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b6dd9bf441d24ffa886996c210b4607f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00013-of-00018.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0fa5351d1be542b88a3024b900b0ef19",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00016-of-00018.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1452efc3d5564ef99ae336dbcc36b22f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00017-of-00018.parquet:   0%|          | 0.00/222M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c12fa5a2d21241c6a31b4c889999ad0a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/2665357 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ca6b84a24ecf4e00bf8e550ba5eb5054",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading dataset shards:   0%|          | 0/18 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 2665357\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.de\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1c60e53d-3031-44e7-b1da-edc419079f13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"de_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c42100db-79d9-4cbe-9582-e43d58567173",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "eee3698d-ccb8-4cec-9796-e93886fd6b1e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a03b4c97410645eaaae45f25e1c03108",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00015.parquet:   0%|          | 0.00/612M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5a03f25cd8354e3299b82554c0fc3287",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00015.parquet:   0%|          | 0.00/383M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8a63110f3cf648eda442ee3177aee2f6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00015.parquet:   0%|          | 0.00/305M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "09ec9ce700084abcb944f6b6669ebf15",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00015.parquet:   0%|          | 0.00/252M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "363a3b6272c24e23a0e708864c872310",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00015.parquet:   0%|          | 0.00/240M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "58b7f596d96246439e7cc21acc333fb4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00015.parquet:   0%|          | 0.00/240M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2872cc14f4944b4894bd05bbb256d892",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00015.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3d65c3b76a02445e85d6723ea19565c2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00015.parquet:   0%|          | 0.00/218M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a77dd5822bd34896b4dbd4d22cb8331c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00015.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "52ce12211c8743acac275af2c7ac049c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00015.parquet:   0%|          | 0.00/235M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9da3788934f14b15a63e4add38e989b4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00015.parquet:   0%|          | 0.00/217M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "06254bec73a24830b685468235260f0c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00015.parquet:   0%|          | 0.00/213M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d25a6d36b72446458525dd060d63102a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00015.parquet:   0%|          | 0.00/201M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4da023af77894a72a1e6223c91820983",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00013-of-00015.parquet:   0%|          | 0.00/193M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6c6d63c3e4df46c69da6b42c8b7eae4a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00014-of-00015.parquet:   0%|          | 0.00/176M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f98c43bad8a140cca38d7c00232f884a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/1389467 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 1389467\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.ja\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "4b61a5b0-85ca-4a0e-99b0-0ea2e3bc9e82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"ja_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5c56f7fd-8188-4d30-928d-e307764f23f9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "1668c12c-cbfe-4ea8-82c6-4e27607e9b67",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8f8cd7bb3992440c89087f809b96832d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00013.parquet:   0%|          | 0.00/688M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b6c6f856ff3f4070a26e56bfacca6deb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00013.parquet:   0%|          | 0.00/376M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d90dc9bb27ef418e906109adf3e7fad3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00013.parquet:   0%|          | 0.00/287M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e72006a2edd24c3a9bd88a0e2ac57d7b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00013.parquet:   0%|          | 0.00/245M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "df667cedb5374cd39e86e52c6cea3a7a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00013.parquet:   0%|          | 0.00/168M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6b005307539743769e0ac974c2996d9b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00013.parquet:   0%|          | 0.00/178M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c749d7a330f24d2fa684857be0700cc3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00013.parquet:   0%|          | 0.00/216M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7b07061ea4694ba8af395eeae31bd1fd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00013.parquet:   0%|          | 0.00/241M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e07a225a3f24d9a9c92ad9ee153b066",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00013.parquet:   0%|          | 0.00/227M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3054fd1470a34bbab1c6dee28dcb2441",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00013.parquet:   0%|          | 0.00/223M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "90f7a09fb95b4c4a926711a258c32107",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00013.parquet:   0%|          | 0.00/167M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f2ceb836cfca47e2ba75659dacfc9739",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00013.parquet:   0%|          | 0.00/254M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b20a7cea4fb146a5810cde198ef60ab6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00013.parquet:   0%|          | 0.00/226M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ba34e3fba4be480cbf145281eb834d64",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/1841155 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 1841155\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.es\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "9b3d2f1d-47c3-4a4d-8f60-cf5874ec643b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"es_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d300900-2d36-4f21-a799-17b466239666",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "1d20d238-3015-443f-956a-ca1a7c622001",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 1384748\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.zh\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "eac16b65-44d1-43ae-9aeb-c06a470daf0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"zh_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67c13fd3-ce11-41ce-8ca9-7d1d783310fb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "8d4f223d-accc-44f5-acf3-5eca7621e960",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2b39607ab45b4449b96d05858de65c90",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00003.parquet:   0%|          | 0.00/400M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c930ca1ff58a4ba2995158c1944335c2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00003.parquet:   0%|          | 0.00/205M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "37654989816342af9a355a2cb6e770b4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00003.parquet:   0%|          | 0.00/177M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b0dbd13bc9dc457eae92438fd63947ae",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/647897 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'url', 'title', 'text'],\n",
+       "        num_rows: 647897\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.ko\")\n",
+    "datasets_wiki"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "1af1e848-559a-4882-a3ce-966a63edac44",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wiki = open(\"ko_wiki.txt\",\"w\")\n",
+    "for item in datasets_wiki[\"train\"]:\n",
+    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
+    "    wiki.write(line)\n",
+    "wiki.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4b3810a-f564-4a4e-8ccc-5f23f8541414",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

train_data/de_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50fa5e0f7dc3d1e7e29b129b380b2ebe2226176e752fa168ce1327e5f4b11200
+size 4354626975

train_data/dna_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3acc05887b61da97e202a564ba99c6b394f260f7bfa4306ba18c2637f40f7757
+size 4052518950

train_data/en_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e910449e76e03672deed6983640b08629c7ccd9f4579812b5eb5663c13ac9bf
+size 3953811499

train_data/es_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b31bbcdea9751517707b52144a133d61a1df3cdba799b0c51dd9b307a6791b09
+size 3928023424

train_data/fr_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e6e85cc891270b855ee6747df6320760ef2c91118d2d7f6df6c80ebc3ef9d1
+size 4794614101

train_data/gene_eng.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0122099bd827924fd85f178305635b81ecaa54f157f2d340e09727a65320faa2
+size 12245273833

train_data/gene_eng_zh.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b73592b4ce380b050cf6e51a9f8b5eed4a82d48f47e17f4bef811b8572a42fdf
+size 14897010606

train_data/gene_eng_zh_de_es.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aed6299fd5c812d09f9c5fe6a9f7f7dec9f6bcc7a4df13703859e19949eefd6
+size 23179661005

train_data/ja_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3e89b190a14ec963fbccf1ef0642c4f8d2b036a6c843974b961176ec0fc3ecd
+size 4585860236

train_data/ko_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f42d67971f7cfbc05b309816ec8063310dac075ec8ca1c166fa0bfd4261f9d7
+size 1344091574

train_data/protein_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0921aa4a0ed580363e67953c09e82785941596f53d9c6b21a6ae6bf298c9f52
+size 4238943384

train_data/zh_wiki_4g.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf18bd3e5917e94e3edac7a3b8bffd73ecbcc77806db69044a29f45591e34958
+size 2651736773