Upload all models and assets for as (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: as
-language_name: AS
+language_name: Assamese
 language_family: indoaryan_eastern
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-indoaryan_eastern
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.534
+    value: 4.542
   - name: best_isotropy
     type: isotropy
-    value: 0.8566
+    value: 0.8547
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# AS - Wikilangs Models
+# Assamese - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **AS** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Assamese** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,10 +90,10 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.446x | 3.45 | 0.0759% | 1,436,355 |
-| **16k** | 3.889x | 3.89 | 0.0856% | 1,272,728 |
-| **32k** | 4.259x | 4.26 | 0.0938% | 1,162,147 |
-| **64k** | 4.534x 🏆 | 4.53 | 0.0999% | 1,091,630 |
+| **8k** | 3.450x | 3.45 | 0.0757% | 1,416,711 |
+| **16k** | 3.894x | 3.89 | 0.0855% | 1,255,391 |
+| **32k** | 4.266x | 4.27 | 0.0937% | 1,145,685 |
+| **64k** | 4.542x 🏆 | 4.54 | 0.0997% | 1,076,075 |
 
 ### Tokenization Examples
 
@@ -98,29 +108,29 @@ Below are sample sentences tokenized with each vocabulary size:
 | 32k | `▁জয়ন গৰ ▁মজ িল পুৰ ▁ভাৰতৰ ▁পশ্চিমবংগ ▁ৰাজ্যৰ ▁দক্ষিণ ▁চব্বিশ ... (+8 more)` | 18 |
 | 64k | `▁জয়নগৰ ▁মজ িল পুৰ ▁ভাৰতৰ ▁পশ্চিমবংগ ▁ৰাজ্যৰ ▁দক্ষিণ ▁চব্বিশ ▁পৰগনা ... (+7 more)` | 17 |
 
-**Sample 2:** `প্ৰদীপ আচাৰ্য্য একবিংশ শতাব্দীৰ অসমৰ এগৰাকী প্ৰসিদ্ধ লেখক, সমালোচক । সংক্ষিপ্ত জ...`
+**Sample 2:** `হাবুং মৈদাম হৈছে আহোমসকলৰ পঞ্চমৰাজধানী হাবুংৰ টাইভেটিত অৱস্থিত দুটা প্ৰাচীন মৈদা...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁প্ৰদীপ ▁আচাৰ্য ্য ▁এক বিংশ ▁শতা ব্দ ীৰ ▁অসমৰ ▁এগৰাকী ... (+10 more)` | 20 |
-| 16k | `▁প্ৰদীপ ▁আচাৰ্য ্য ▁একবিংশ ▁শতাব্দীৰ ▁অসমৰ ▁এগৰাকী ▁প্ৰসিদ্ধ ▁লেখক , ... (+7 more)` | 17 |
-| 32k | `▁প্ৰদীপ ▁আচাৰ্য ্য ▁একবিংশ ▁শতাব্দীৰ ▁অসমৰ ▁এগৰাকী ▁প্ৰসিদ্ধ ▁লেখক , ... (+7 more)` | 17 |
-| 64k | `▁প্ৰদীপ ▁আচাৰ্য ্য ▁একবিংশ ▁শতাব্দীৰ ▁অসমৰ ▁এগৰাকী ▁প্ৰসিদ্ধ ▁লেখক , ... (+7 more)` | 17 |
+| 8k | `▁হাব ু ং ▁মৈ দ াম ▁হৈছে ▁আহোম সকলৰ ▁পঞ্চম ... (+31 more)` | 41 |
+| 16k | `▁হাব ুং ▁মৈ দাম ▁হৈছে ▁আহোম সকলৰ ▁পঞ্চম ৰাজ ধান ... (+26 more)` | 36 |
+| 32k | `▁হাব ুং ▁মৈদাম ▁হৈছে ▁আহোমসকলৰ ▁পঞ্চম ৰাজ ধানী ▁হাব ুং ... (+21 more)` | 31 |
+| 64k | `▁হাবুং ▁মৈদাম ▁হৈছে ▁আহোমসকলৰ ▁পঞ��চম ৰাজধানী ▁হাবুং ৰ ▁টাই ভেটিত ... (+16 more)` | 26 |
 
-**Sample 3:** `মাটিকালি অৱস্থান কৰ্মচাৰী সা-সুবিধা তথ্যসূত্ৰ বিদ্যালয়সমূহ`
+**Sample 3:** `ভাৰতীয় ন্যায় সংহিতা (IAST: Bhāratīya Nyāya Saṃhitā), ভাৰতীয় গণৰাজ্যৰ অপৰাধ সং...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁মাটিকালি ▁অৱস্থান ▁কৰ্মচাৰী ▁সা - সু বিধা ▁তথ্যসূত্ৰ ▁বিদ্যালয় সমূহ` | 10 |
-| 16k | `▁মাটিকালি ▁অৱস্থান ▁কৰ্মচাৰী ▁সা - সু বিধা ▁তথ্যসূত্ৰ ▁বিদ্যালয়সমূহ` | 9 |
-| 32k | `▁মাটিকালি ▁অৱস্থান ▁কৰ্মচাৰী ▁সা - সুবিধা ▁তথ্যসূত্ৰ ▁বিদ্যালয়সমূহ` | 8 |
-| 64k | `▁মাটিকালি ▁অৱস্থান ▁কৰ্মচাৰী ▁সা - সুবিধা ▁তথ্যসূত্ৰ ▁বিদ্যালয়সমূহ` | 8 |
+| 8k | `▁ভাৰতীয় ▁ন্যায় ▁সংহ িতা ▁( i ast : ▁bh ā ... (+27 more)` | 37 |
+| 16k | `▁ভাৰতীয় ▁ন্যায় ▁সংহিতা ▁( i ast : ▁bh ā rat ... (+23 more)` | 33 |
+| 32k | `▁ভাৰতীয় ▁ন্যায় ▁সংহিতা ▁( iast : ▁bh ā rat ī ... (+20 more)` | 30 |
+| 64k | `▁ভাৰতীয় ▁ন্যায় ▁সংহিতা ▁( iast : ▁bh ā rat īya ... (+18 more)` | 28 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.534x compression
-- **Lowest UNK Rate:** 8k with 0.0759% unknown tokens
+- **Best Compression:** 64k achieves 4.542x compression
+- **Lowest UNK Rate:** 8k with 0.0757% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 60,931 | 15.89 | 198,049 | 8.3% | 21.5% |
-| **2-gram** | Subword | 2,317 🏆 | 11.18 | 62,544 | 34.0% | 69.3% |
-| **3-gram** | Word | 105,867 | 16.69 | 226,215 | 4.9% | 14.7% |
-| **3-gram** | Subword | 21,008 | 14.36 | 364,128 | 13.2% | 35.4% |
-| **4-gram** | Word | 237,754 | 17.86 | 355,974 | 2.4% | 7.8% |
-| **4-gram** | Subword | 113,775 | 16.80 | 1,477,005 | 7.8% | 20.9% |
+| **2-gram** | Word | 62,472 | 15.93 | 206,764 | 8.3% | 21.4% |
+| **2-gram** | Subword | 2,308 🏆 | 11.17 | 63,567 | 34.0% | 69.4% |
+| **3-gram** | Word | 109,754 | 16.74 | 237,526 | 5.0% | 14.6% |
+| **3-gram** | Subword | 20,939 | 14.35 | 371,943 | 13.3% | 35.5% |
+| **4-gram** | Word | 247,178 | 17.92 | 371,701 | 2.3% | 7.7% |
+| **4-gram** | Subword | 113,780 | 16.80 | 1,515,602 | 7.8% | 20.9% |
+| **5-gram** | Word | 182,489 | 17.48 | 239,039 | 1.9% | 7.3% |
+| **5-gram** | Subword | 319,720 | 18.29 | 2,664,609 | 5.1% | 14.4% |
 
 ### Top 5 N-grams by Size
 
@@ -150,68 +162,88 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `কৰা হয়` | 27,116 |
-| 2 | `কৰা হৈছিল` | 11,596 |
-| 3 | `হ ল` | 10,746 |
-| 4 | `লাভ কৰে` | 10,053 |
-| 5 | `কৰা হৈছে` | 9,448 |
+| 1 | `কৰা হয়` | 29,188 |
+| 2 | `কৰা হৈছিল` | 12,508 |
+| 3 | `হ ল` | 11,276 |
+| 4 | `লাভ কৰে` | 10,608 |
+| 5 | `কৰা হৈছে` | 10,201 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ব্যৱহাৰ কৰা হয়` | 3,039 |
-| 2 | `হ ব পাৰে` | 3,023 |
-| 3 | `বুলি কোৱা হয়` | 2,966 |
-| 4 | `গণ্য কৰা হয়` | 2,121 |
-| 5 | `ডিগ্ৰী লাভ কৰে` | 1,927 |
+| 1 | `ব্যৱহাৰ কৰা হয়` | 3,336 |
+| 2 | `হ ব পাৰে` | 3,197 |
+| 3 | `বুলি কোৱা হয়` | 3,190 |
+| 4 | `গণ্য কৰা হয়` | 2,309 |
+| 5 | `ডিগ্ৰী লাভ কৰে` | 2,043 |
 
 **4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `তথ্য সংগ্ৰহ বাহ্যিক সংযোগ` | 1,636 |
-| 2 | `বুলি গণ্য কৰা হয়` | 1,147 |
-| 3 | `স্নাতক ডিগ্ৰী লাভ কৰে` | 819 |
-| 4 | `তথ্য উৎস বাহ্যিক সংযোগ` | 772 |
-| 5 | `হিচাপে গণ��য কৰা হয়` | 749 |
+| 1 | `তথ্য সংগ্ৰহ বাহ্যিক সংযোগ` | 1,641 |
+| 2 | `বুলি গণ্য কৰা হয়` | 1,265 |
+| 3 | `স্নাতক ডিগ্ৰী লাভ কৰে` | 864 |
+| 4 | `হিচাপে গণ্য কৰা হয়` | 801 |
+| 5 | `তথ্য উৎস বাহ্যিক সংযোগ` | 782 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `archived from the original on` | 423 |
+| 2 | `অভিনেত্ৰী চলচ্চিত্ৰৰ অভিনেত্ৰী চলচ্চিত্ৰৰ অভিনেত্ৰী` | 245 |
+| 3 | `দিনটোত ঘটা কেইটামান উল্লেখযোগ্য ঘটনা` | 244 |
+| 4 | `এই দিনটোত ঘটা কেইটামান উল্লেখযোগ্য` | 237 |
+| 5 | `প্ৰাৰম্ভিক জীৱন আৰু শিক্ষা চনৰ` | 214 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `ৰ _` | 1,253,155 |
-| 2 | `ত _` | 617,790 |
-| 3 | `_ আ` | 557,646 |
-| 4 | `। _` | 441,423 |
-| 5 | `_ ক` | 431,976 |
+| 1 | `ৰ _` | 1,309,192 |
+| 2 | `ত _` | 645,783 |
+| 3 | `_ আ` | 585,970 |
+| 4 | `। _` | 462,800 |
+| 5 | `_ ক` | 454,528 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `আ ৰু _` | 234,191 |
-| 2 | `_ আ ৰু` | 234,020 |
-| 3 | `_ ক ৰি` | 132,035 |
-| 4 | `_ তে ওঁ` | 130,105 |
-| 5 | `ন ৰ _` | 119,581 |
+| 1 | `আ ৰু _` | 247,371 |
+| 2 | `_ আ ৰু` | 247,194 |
+| 3 | `_ ক ৰি` | 139,299 |
+| 4 | `_ তে ওঁ` | 136,655 |
+| 5 | `ন ৰ _` | 124,787 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ আ ৰু _` | 233,600 |
-| 2 | `ছি ল । _` | 95,977 |
-| 3 | `_ ক ৰা _` | 84,715 |
-| 4 | `_ তে ওঁ _` | 61,201 |
-| 5 | `_ এ ই _` | 61,142 |
+| 1 | `_ আ ৰু _` | 246,758 |
+| 2 | `ছি ল । _` | 101,454 |
+| 3 | `_ ক ৰা _` | 90,139 |
+| 4 | `_ এ ই _` | 64,252 |
+| 5 | `_ তে ওঁ _` | 64,067 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ হ য় । _` | 58,520 |
+| 2 | `_ ক ৰে । _` | 51,805 |
+| 3 | `_ ক ৰি ছি ল` | 51,692 |
+| 4 | `ৰ _ বা বে _` | 47,193 |
+| 5 | `_ চ ন ত _` | 47,011 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 2,317
+- **Best Perplexity:** 2-gram (subword) with 2,308
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~21% of corpus
+- **Coverage:** Top-1000 patterns cover ~14% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.8462 | 1.798 | 7.80 | 533,621 | 15.4% |
-| **1** | Subword | 0.8352 | 1.784 | 12.14 | 14,852 | 16.5% |
-| **2** | Word | 0.2679 | 1.204 | 1.70 | 4,157,114 | 73.2% |
-| **2** | Subword | 0.7097 | 1.635 | 5.34 | 180,337 | 29.0% |
-| **3** | Word | 0.0819 | 1.058 | 1.15 | 7,059,669 | 91.8% |
-| **3** | Subword | 0.5596 | 1.474 | 3.48 | 962,394 | 44.0% |
-| **4** | Word | 0.0273 🏆 | 1.019 | 1.04 | 8,117,676 | 97.3% |
-| **4** | Subword | 0.4358 | 1.353 | 2.26 | 3,350,979 | 56.4% |
+| **1** | Word | 0.8450 | 1.796 | 7.84 | 550,295 | 15.5% |
+| **1** | Subword | 0.8398 | 1.790 | 12.10 | 15,252 | 16.0% |
+| **2** | Word | 0.2695 | 1.205 | 1.71 | 4,311,912 | 73.1% |
+| **2** | Subword | 0.7069 | 1.632 | 5.33 | 184,530 | 29.3% |
+| **3** | Word | 0.0827 | 1.059 | 1.15 | 7,360,379 | 91.7% |
+| **3** | Subword | 0.5599 | 1.474 | 3.49 | 984,248 | 44.0% |
+| **4** | Word | 0.0276 🏆 | 1.019 | 1.04 | 8,480,563 | 97.2% |
+| **4** | Subword | 0.4373 | 1.354 | 2.27 | 3,437,009 | 56.3% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `আৰু আন্তঃৰাষ্ট্ৰীয় ত্ৰিবৰ্ষীয় নতুন আলোচনী বিভাগ আৰু তিনিটা ভাগত কেইটামান ষ্ট্ৰ মেল ফেৰাৰ হেলেনা দ্...`
-2. `কৰা আয়াতসমূহক সাধাৰণতে এই লিংগ জাতীয় উৎসৱ পৰ্ব অনুষ্ঠান হিচাপে ব্যৱহাৰ সংস্কৃতিক কেন্দ্ৰৰ centre s...`
-3. `হয় মিনিক থিয়েম ৬ ৩৬ চৌৰঙ্গী উপন্যাসৰ নতুন ব্ৰডগজ ইঞ্জিন বিদ্যুতৰ অনুমতি দিযা নি পকড় কাফীৰ`
+1. `আৰু লেখিকা বুলি সকলোৱে উৎসাহিত কৰাৰ উদ্দেশ্যে চনত বামুণপাৰা বালিপাৰা ষ্টীম কুকাৰতে খাদ্যৰ ৬৫ মিলিয়ন...`
+2. `কৰা এক শিক্ষা প্ৰদানৰ বিষয় হিচাপে কাৰ্যনিৰ্বাহ কৰিছিল মৃত্যু চনত হোমেন বৰগোহাঞিৰ এখন মেল পাতে আৰু`
+3. `হয় আমেৰিকা যুক্তৰাষ্টৰ প্ৰথম উপাচাৰ্য আছিল অনুমান কৰা পুৰণি অট্টালিকাবোৰত মধ্যমীয়া চৰিত্ৰত অভিনয় ...`
 
 **Context Size 2:**
 
-1. `কৰা হয় পিছলৈ কানৱ ঋষিৰ আশ্ৰমত বাস কৰে স্থিতি তথা সংৰক্ষণ তথ্যসূত্ৰ বহিঃসংযোগ ইণ্টাৰনেট মুভি ডাটাবেছ...`
-2. `কৰা হৈছিল ৰডবোৰে এটা সংখ্যাৰ অংকৰ যোগফলৰ দ্বাৰা পোৱা গৈছিল উদ্ভিদ ৰসায়ন টিনোস্প ৰা কৰ্ডিফ লিয়াত এল...`
-3. `হ ল ছিৰাম চিক্‌নেছ সদৃশ লক্ষণ প্ৰদৰ্শনকাৰী মধ্যমীয়া আৰু যুক্তিসংগত বিশ্লেষণৰ জৰিয়তে শ্ৰীকৃষ্ণ কীৰ্...`
+1. `কৰা হয় চনৰ জানুৱাৰী মাহত অম্বা বহোৰা নামৰ এগৰাকী যুৱতীক তেওঁৰ স্বামী টোপনি যোৱালৈকে অপেক্ষা কৰাটো প...`
+2. `কৰা হৈছিল কলহোৰা শাসকসকলৰ সমাধিস্থলত ফুল আৰু প্ৰসাদেৰে তুলসীক পূজা কৰা ধৰণৰ তাৰতম্য আছিল তথাপি ধৰ্মে...`
+3. `হ ল পদ্মভূষণ ভাৰতৰ তৃতীয় সৰ্বোচ্চ অসামৰিক সন্মান পদ্মশ্ৰী লাভ কৰে তেখেতে অভিনয় কৰে চনত তেওঁৰ নিজাক...`
 
 **Context Size 3:**
 
-1. `ব্যৱহাৰ কৰা হয় এটা জনপ্ৰিয় কিংবদন্তি অনুসৰি বৈষ্ণৱ পণ্ডিতসকলে শৈৱ বুলি প্ৰত্যাখ্যান কৰাৰ পিছত তেওঁ...`
-2. `হ ব পাৰে ৰচনাৰ তাৰিখ ঐতৰেয় ব্ৰাহ্মণ কিছু নিশ্চিতভাৱে খ্ৰীষ্টপূৰ্ব ১ম সহস্ৰাব্দৰ সম্ভৱতঃ ইয়াৰ প্ৰথম...`
-3. `বুলি কোৱা হয় চনৰ ১১ ফেব্ৰুৱাৰীত বাংলাদেশত আলিৰ মৃত্যু হয় তেওঁৰ মৃত্যুৰ পিছত নিউয়ৰ্ক টাইমছে তেওঁক ...`
+1. `ব্যৱহাৰ কৰা হয় msa এ smtp প্ৰটোকলত প্ৰদান কৰা গন্তব্যস্থানৰ ঠিকনা নিৰ্ধাৰণ কৰে বাৰ্তা হেডাৰৰ পৰা নহ...`
+2. `হ ব পাৰে অসমৰ কবি লেখক জীৱন নৰহে আত্মজীৱনীমূলক গ্ৰন্থখনক নতুন প্ৰজন্মৰ সাহসৰ দলিল বুলি অভিহিত কৰে অৰ...`
+3. `বুলি কোৱা হয় ৰাক্ষসসকলক প্ৰায় পৰাধীন সৈনিকৰ ৰূপত দেখুৱা হৈছিল পিছে কিছু ৰাক্ষসে অত্যন্ত বল অৰ্জন ক...`
 
 **Context Size 4:**
 
-1. `তথ্য সংগ্ৰহ বাহ্যিক সংযোগ আনুষ্ঠানিক mamata banerjee official all india trinamool congress party pro...`
-2. `বুলি গণ্য কৰা হয় একশৰণ নাম ধৰ্মৰ অনুগামীসকলে গুণমালা পুথিখনক অতি পৱিত্ৰ জ্ঞান কৰি গুৰু আসনত প্ৰতিষ্...`
-3. `স্নাতক ডিগ্ৰী লাভ কৰে আৰু বেৰিষ্টাৰ ইজাজ হুছেইন বাটালৱীৰ চেম্বাৰত যোগদান কৰে চনত লাহোৰ উচ্চ ন্যায়াল...`
+1. `তথ্য সংগ্ৰহ বাহ্যিক সংযোগ cornell university e book library of classic texts on mechanical design an...`
+2. `বুলি গণ্য কৰা হয় আৰু কোমল গ আৰু কোমল ধ স্বৰসমূহ কম্পনৰ সৈতে অন্দোলিত পৰিবেশিত হয় সকলো পাঁচটা স্বৰ`
+3. `স্নাতক ডিগ্ৰী লাভ কৰে সেই একেই দীন দয়াল উপাধ্যায় কলেজৰ পৰা সামাজিক কাম চনত ১৯ বছৰ বয়সত ছেম অল্টমে...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_ভাৱেশনত-_পলবাদলকৰা_`
-2. `ৰ_বিয়া_usis_ই_সেই_ধ্ব`
-3. `কবৰু_।_মহিলাৰশ্বি_ম_৩৬`
+1. `_ধান_লা_শক্তি।_হাসিদ্ধ_তাকা`
+2. `ৰক্ষ_নিজনগোৱাহালচ্চিত্ৰ_মবাবে`
+3. `কথাই_ইছিল_জীৱন_ডিয়_বা`
 
 **Context Size 2:**
 
-1. `ৰ_ডে'_+_(spishaver`
-2. `ত_উপন্যাসৰ_ওপৰ_ওৰে_চক্ৰ`
-3. `_আৰু_পাৰে।_চলন_বোমা-কাশ্মী`
+1. `ৰ_আৰু_লাউ।_অসংখ্যক_ভাৰত`
+2. `ত_জ্ঞানৰ_কৃপ,_কেন্দ্ৰটোৰ_পৰা`
+3. `_আৰু_মোৰ_পৰা_উৰুলিয়ান_শা`
 
 **Context Size 3:**
 
-1. `আৰু_মৰলৰ_মেক্সিমফৰ_এটা-আ`
-2. `_আৰু_লাইকা"_এটা_উত্তৰ_শ্ৰেষ্ঠ`
-3. `_কৰিবলৈ_গঢ়_লৈ_যোৱা_সমসাম`
+1. `আৰু_বীৰেন্দ্ৰ_মোডীৰ_নিগমৰ_প্ৰয়া`
+2. `_আৰু_অৰ্থ।_দেৱালয়খনৰ_পিছ`
+3. `_কৰিবলৈ_অস্বীকাৰ_হোৱা_মতবাদ`
 
 **Context Size 4:**
 
-1. `_আৰু_সামাজিক_অৱ_ছিংগাপুৰত_২`
-2. `ছিল।_ইভান্সে_প্ৰাণীবিজ্ঞান,_ক্ৰমবৰ্ধ`
-3. `_কৰা_হয়।_ষ্টাফ_ৰিপৰ্টাৰ_২৪_`
+1. `_আৰু_পাম_তেল_আৰু_পিপিপি_আৰু`
+2. `ছিল।_কুমাৰীত্ব_পৰীক্ষাৰ_অংগ_আৰু`
+3. `_কৰা_দুখৰ_আৰু_তেওঁৰ_ছিলভাৰ`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 (word) with 97.3% predictability
+- **Best Predictability:** Context-4 (word) with 97.2% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (3,350,979 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (3,437,009 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 219,027 |
-| Total Tokens | 8,615,852 |
-| Mean Frequency | 39.34 |
+| Vocabulary Size | 225,407 |
+| Total Tokens | 9,007,362 |
+| Mean Frequency | 39.96 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 763.95 |
+| Frequency Std Dev | 792.95 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | আৰু | 234,273 |
-| 2 | কৰা | 88,269 |
-| 3 | হয় | 83,006 |
-| 4 | কৰে | 74,637 |
-| 5 | এই | 61,800 |
-| 6 | তেওঁ | 61,727 |
-| 7 | পৰা | 52,844 |
-| 8 | কৰিছিল | 48,735 |
-| 9 | বাবে | 48,165 |
-| 10 | চনত | 47,181 |
+| 1 | আৰু | 247,463 |
+| 2 | কৰা | 93,923 |
+| 3 | হয় | 87,716 |
+| 4 | কৰে | 78,599 |
+| 5 | এই | 64,931 |
+| 6 | তেওঁ | 64,613 |
+| 7 | পৰা | 53,636 |
+| 8 | কৰিছিল | 51,623 |
+| 9 | বাবে | 50,799 |
+| 10 | চনত | 49,149 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | চেকিজাং | 2 |
-| 2 | জটৱানী | 2 |
-| 3 | জটৱানীৰ | 2 |
-| 4 | ভিটাইৰ | 2 |
-| 5 | সিন্ধীজ | 2 |
-| 6 | দেৱচন্দ্ৰৰ | 2 |
-| 7 | দেৱচন্দ্ৰ | 2 |
-| 8 | প্ৰাণনাথৰ | 2 |
-| 9 | প্ৰাণনাথে | 2 |
-| 10 | গুৰদ্বাৰ | 2 |
+| 1 | পাণ্ডুবংশী | 2 |
+| 2 | মনুমেণ্টছ | 2 |
+| 3 | ছিৰপুৰৰ | 2 |
+| 4 | swfl | 2 |
+| 5 | manhunt | 2 |
+| 6 | megamodel | 2 |
+| 7 | গ্লেডৰেগ্‌চ | 2 |
+| 8 | কিস | 2 |
+| 9 | পদাইভীৰণ | 2 |
+| 10 | বিগিল | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0086 |
-| R² (Goodness of Fit) | 0.989742 |
+| Zipf Coefficient | 1.0094 |
+| R² (Goodness of Fit) | 0.989782 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 25.4% |
-| Top 1,000 | 50.8% |
-| Top 5,000 | 71.8% |
-| Top 10,000 | 79.6% |
+| Top 100 | 25.5% |
+| Top 1,000 | 50.9% |
+| Top 5,000 | 71.9% |
+| Top 10,000 | 79.7% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9897 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 25.4% of corpus
-- **Long Tail:** 209,027 words needed for remaining 20.4% coverage
+- **Zipf Compliance:** R²=0.9898 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 25.5% of corpus
+- **Long Tail:** 215,407 words needed for remaining 20.3% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8476 | 0.3643 | N/A | N/A |
-| **mono_64d** | 64 | 0.8566 🏆 | 0.2729 | N/A | N/A |
-| **mono_128d** | 128 | 0.8399 | 0.2134 | N/A | N/A |
+| **mono_32d** | 32 | 0.8458 | 0.3637 | N/A | N/A |
+| **mono_64d** | 64 | 0.8547 🏆 | 0.2742 | N/A | N/A |
+| **mono_128d** | 128 | 0.8359 | 0.2093 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8458 | 0.3735 | 0.0580 | 0.3060 |
+| **aligned_64d** | 64 | 0.8547 | 0.2836 | 0.1180 | 0.3960 |
+| **aligned_128d** | 128 | 0.8359 | 0.2075 | 0.1480 | 0.4820 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_64d with 0.8566 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2836. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_64d with 0.8547 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2853. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 14.8% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.495** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -430,8 +465,8 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-ৰ` | নিৰ্ধাৰনৰ, স্পঞ্জৰ, চেমেলেৰ |
-| `-াৰ` | শীতলাৰ, গুজ্জাৰ, ভেঁ‌ড়াৰ |
+| `-ৰ` | কেথলিকসকলৰ, স্বপ্ৰচাৰ, ৱিণ্টাৰ |
+| `-াৰ` | স্বপ্ৰচাৰ, ৱিণ্টাৰ, অফকাটাৰ |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -439,18 +474,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `ther` | 3.32x | 64 contexts | other, theri, there |
-| `ight` | 3.29x | 55 contexts | bight, tight, might |
-| `ress` | 3.32x | 41 contexts | dress, press, presse |
-| `indi` | 3.32x | 39 contexts | hindi, indie, pindi |
-| `vers` | 3.14x | 46 contexts | evers, overs, verse |
-| `nter` | 3.21x | 38 contexts | inter, enter, hunter |
-| `olog` | 3.28x | 34 contexts | oology, biology, zoology |
-| `ment` | 3.17x | 38 contexts | cement, mentor, mentha |
-| `ctio` | 3.34x | 31 contexts | action, diction, section |
-| `atio` | 3.18x | 37 contexts | fatio, ratio, nation |
-| `stor` | 3.19x | 33 contexts | storm, jstor, story |
-| `iver` | 3.17x | 26 contexts | liver, river, giver |
+| `ther` | 3.36x | 64 contexts | theri, there, other |
+| `ress` | 3.34x | 44 contexts | press, dress, duress |
+| `nter` | 3.33x | 38 contexts | inter, enter, wynter |
+| `vers` | 3.15x | 47 contexts | verso, versa, verse |
+| `atio` | 3.32x | 37 contexts | ratio, fatio, nation |
+| `indi` | 3.24x | 39 contexts | hindi, indie, india |
+| `ment` | 3.24x | 38 contexts | cement, moment, mental |
+| `stor` | 3.25x | 35 contexts | storm, jstor, story |
+| `ctio` | 3.33x | 32 contexts | action, auction, faction |
+| `iver` | 3.16x | 26 contexts | liver, giver, river |
+| `ersi` | 3.21x | 20 contexts | persia, persie, yersin |
+| `mber` | 3.17x | 18 contexts | amber, number, member |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -465,26 +500,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| লিথুৱানিয়াৰ | **`লিথুৱানিয়-াৰ`** | 1.5 | `লিথুৱানিয়` |
-| পুনৰ্ব্যৱহাৰ | **`পুনৰ্ব্যৱহ-াৰ`** | 1.5 | `পুনৰ্ব্যৱহ` |
-| প্লাছিয়াৰ | **`প্লাছিয়-াৰ`** | 1.5 | `প্লাছিয়` |
-| ক্ষেত্ৰাধিকাৰ | **`ক্ষেত্ৰাধিক-াৰ`** | 1.5 | `ক্ষেত্ৰাধিক` |
-| বিন্ধোৱাৰ | **`বিন্ধোৱ-াৰ`** | 1.5 | `বিন্ধোৱ` |
-| চুলিক্‌ফাৰ | **`চুলিক্‌ফ-াৰ`** | 1.5 | `চুলিক্‌ফ` |
-| ইউনিলিভাৰ | **`ইউনিলিভ-াৰ`** | 1.5 | `ইউনিলিভ` |
-| চিৰস্তাদাৰ | **`চিৰস্তাদ-াৰ`** | 1.5 | `চিৰস্তাদ` |
-| লাখটকীয়াৰ | **`লাখটকীয়-াৰ`** | 1.5 | `লাখটকীয়` |
-| জাতিসত্তাৰ | **`জাতিসত্ত-াৰ`** | 1.5 | `জাতিসত্ত` |
-| দৰিদ্ৰতাৰ | **`দৰিদ্ৰত-াৰ`** | 1.5 | `দৰিদ্ৰত` |
-| ছিলভেষ্টাৰ | **`ছিলভেষ্ট-াৰ`** | 1.5 | `ছিলভেষ্ট` |
-| চিলভেষ্টাৰ | **`চিলভেষ্ট-াৰ`** | 1.5 | `চিলভেষ্ট` |
-| বাগ্মীতাৰ | **`বাগ্মীত-াৰ`** | 1.5 | `বাগ্মীত` |
-| নিয়ন্ত্ৰণহীনতাৰ | **`নিয়ন্ত্ৰণহীনত-াৰ`** | 1.5 | `নিয়ন্ত্ৰণহীনত` |
+| চেটেছুয়াৰাৰ | **`চেটেছুয়-াৰ-াৰ`** | 3.0 | `চেটেছুয়` |
+| সীতাৰামায়াৰ | **`সীতাৰামায়-াৰ`** | 1.5 | `সীতাৰামায়` |
+| ৰাজ্কুমাৰ | **`ৰাজ্কুম-াৰ`** | 1.5 | `ৰাজ্কুম` |
+| প্ৰতিৰক্ষাৰ | **`প্ৰতিৰক্ষ-াৰ`** | 1.5 | `প্ৰতিৰক্ষ` |
+| দত্তবৰুৱাৰ | **`দত্তবৰুৱ-াৰ`** | 1.5 | `দত্তবৰুৱ` |
+| বিষ্ণুৰাভাৰ | **`বিষ্ণুৰাভ-াৰ`** | 1.5 | `বিষ্ণুৰাভ` |
+| বদৌপায়াৰ | **`বদৌপায়-াৰ`** | 1.5 | `বদৌপায়` |
+| হাছলেংগাৰ | **`হাছলেংগ-াৰ`** | 1.5 | `হাছলেংগ` |
+| চিজাৰিয়াৰ | **`চিজাৰিয়-াৰ`** | 1.5 | `চিজাৰিয়` |
+| কুকুৰাঝাৰ | **`কুকুৰাঝ-াৰ`** | 1.5 | `কুকুৰাঝ` |
+| মন্দাৱস্থাৰ | **`মন্দাৱস্থ-াৰ`** | 1.5 | `মন্দাৱস্থ` |
+| আত্মপ্ৰতিষ্ঠাৰ | **`আত্মপ্ৰতিষ্ঠ-াৰ`** | 1.5 | `আত্মপ্ৰতিষ্ঠ` |
+| কেঁচাগোল্লাৰ | **`কেঁচাগোল্ল-াৰ`** | 1.5 | `কেঁচাগোল্ল` |
+| ফ্ৰণ্টিয়াৰ | **`ফ্ৰণ্টিয়-াৰ`** | 1.5 | `ফ্ৰণ্টিয়` |
+| যিহোচূৱাৰ | **`যিহোচূৱ-াৰ`** | 1.5 | `যিহোচূৱ` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language AS appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Assamese shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -495,9 +530,9 @@ The language AS appears to be more isolating or has a highly fixed vocabulary. W
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (4.53x) |
-| N-gram | **2-gram** | Lowest perplexity (2,317) |
-| Markov | **Context-4** | Highest predictability (97.3%) |
+| Tokenizer | **64k BPE** | Best compression (4.54x) |
+| N-gram | **2-gram** | Lowest perplexity (2,308) |
+| Markov | **Context-4** | Highest predictability (97.2%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
 
@@ -711,4 +746,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 05:56:00*
+*Report Date: 2026-01-03 17:31:44*

models/embeddings/aligned/as_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcc1d48eacb23fa09677a13e7ad69f9512aaa13f84a4ba39f7a2c85dfc093db6
+size 1138420082

models/embeddings/aligned/as_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "as", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/as_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f618208c85e69b0c2f9a9c24fe14deb1d921c09cbd472e2f6ba098921d273c25
+size 65664

models/embeddings/aligned/as_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "as",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 16253,
+  "vocab_size": 108712
+}

models/embeddings/aligned/as_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c1f38d8f3adf49e702cdca3e18b1b494499f0b99c7a801d685a13d2d60561aa
+size 286929266

models/embeddings/aligned/as_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "as", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/as_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61be07664c7af5be2e4ab411763491610406d567c312a6ec2b823c2c438fda38
+size 4224

models/embeddings/aligned/as_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "as",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 16253,
+  "vocab_size": 108712
+}

models/embeddings/aligned/as_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d093ab9a39e11d3235dc4002ecdf3a03777a542827c2618313dd5e688db0e50
+size 570759538

models/embeddings/aligned/as_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "as", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/as_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8de60160552742b93b7dc6657679da8244b4ea7923b990c0737ae1e202398069
+size 16512

models/embeddings/aligned/as_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "as",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 16253,
+  "vocab_size": 108712
+}

models/embeddings/monolingual/as_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ac87d06ad44c5f86054b2229ad119a6da7bf6318454bddbdc55c7518d8c4fed
-size 1134845578
+oid sha256:dcc1d48eacb23fa09677a13e7ad69f9512aaa13f84a4ba39f7a2c85dfc093db6
+size 1138420082

models/embeddings/monolingual/as_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 105317
+  "vocab_size": 108712
 }

models/embeddings/monolingual/as_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d92a65e15cacfccf9d549fee6b6a0ce808975a3c742f0ba231c9f7d4e6b2be4
-size 285962122
+oid sha256:8c1f38d8f3adf49e702cdca3e18b1b494499f0b99c7a801d685a13d2d60561aa
+size 286929266

models/embeddings/monolingual/as_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 105317
+  "vocab_size": 108712
 }

models/embeddings/monolingual/as_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:105129e3ce95f6d925453fd6d253ba75069fd36adaf12abf53d323c8a32ee134
-size 568923274
+oid sha256:6d093ab9a39e11d3235dc4002ecdf3a03777a542827c2618313dd5e688db0e50
+size 570759538

models/embeddings/monolingual/as_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 105317
+  "vocab_size": 108712
 }

models/subword_markov/as_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c80c8a3d77091dfe2da29d02e66a930b780f77dce7362509695133b8da1ea80
-size 1182447
+oid sha256:dad7c1b11d03f5fb8f84b46470d948c6c1b3b69562ab1dbb7d3673412b569699
+size 1204126

models/subword_markov/as_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "as",
-  "unique_contexts": 14852,
-  "total_transitions": 39716105
+  "unique_contexts": 15252,
+  "total_transitions": 41482191
 }

models/subword_markov/as_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:885badaceaa6a76e70eec40f59892815b915e219ae00559881567d3b989d3ef2
-size 7340754
+oid sha256:0867ca2773731731378c0d533277909d409836d815c21b5eadca437ec98143aa
+size 7509153

models/subword_markov/as_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "as",
-  "unique_contexts": 180337,
-  "total_transitions": 39696289
+  "unique_contexts": 184530,
+  "total_transitions": 41461482
 }

models/subword_markov/as_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa1ba33509e5c790fdf467b449165c1f4accd50771e716b68c45863725e8653c
-size 28776160
+oid sha256:3e3d72a978bcda8b0915297039ccce0cf2fe1e7db38e3622d1e7b4acb6e63623
+size 29396511

models/subword_markov/as_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "as",
-  "unique_contexts": 962394,
-  "total_transitions": 39676473
+  "unique_contexts": 984248,
+  "total_transitions": 41440773
 }

models/subword_markov/as_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3c3fed04aa29c942baa4ca8eff754698cf97cb5a32b6ba2a80b814a80269e95
-size 82473540
+oid sha256:e9d1e6df59e244ff52a2fa13ed623028f3c1801221b8785d4c9716d0f72d7505
+size 84826112

models/subword_markov/as_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "as",
-  "unique_contexts": 3350979,
-  "total_transitions": 39656657
+  "unique_contexts": 3437009,
+  "total_transitions": 41420064
 }

models/subword_ngram/as_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:305bff8b4892b68a20e77689ddf5bbd66be7385e136f504ab07ed4ec09661b44
-size 935875
+oid sha256:a407b1bf32a9531bfea8247ab2474244bb4c165cad3e06d61c61a7f7476c1851
+size 952234

models/subword_ngram/as_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "as",
-  "unique_ngrams": 62544,
-  "total_ngrams": 39716105
+  "unique_ngrams": 63567,
+  "total_ngrams": 41482191
 }

models/subword_ngram/as_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67472e7d9683c3e8802771c68e0afe1ec9b79cf06fa64d62f682208c1565fa8f
-size 5569851
+oid sha256:6be9dd12a39b59d055bc880937854749c0c23decef34ce34d282a0e9ec4b4d69
+size 5705561

models/subword_ngram/as_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "as",
-  "unique_ngrams": 364128,
-  "total_ngrams": 39696289
+  "unique_ngrams": 371943,
+  "total_ngrams": 41461482
 }

models/subword_ngram/as_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c4153dc76e2de363499d7501db843a99bc9685c1f78f1712d49767c2d3edb5e
-size 23557608
+oid sha256:a783dfd64b75e5ab0c8dbd9bc2bb9251e0ee16bba614d7621274601fa3c70b11
+size 24184538

models/subword_ngram/as_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "as",
-  "unique_ngrams": 1477005,
-  "total_ngrams": 39676473
+  "unique_ngrams": 1515602,
+  "total_ngrams": 41440773
 }

models/subword_ngram/as_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43b510994c6484c187f3a060ff158e21b1d34dee25dc7d1ae061aec8ccb8e667
+size 45338859

models/subword_ngram/as_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "as",
+  "unique_ngrams": 2664609,
+  "total_ngrams": 41420064
+}

models/tokenizer/as_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75647425d4994600927e38e762eb4e12bd76a7d759b998a6b27d3ef72fb92134
-size 629826
+oid sha256:3abf3b1470b91a2df59ae758d425fcc24465ae66b2371d3e9f9cabdeb3c26232
+size 629337

models/tokenizer/as_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d675824377e9be300eedb755a0221aa145be09b86d891593a1c2428484b63864
-size 1041081
+oid sha256:43bc15224ea4b1ccc61b70c25f15ae6b8f19f47cc21ee1ddd0d01f33c1aba079
+size 1039065

models/tokenizer/as_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3a5aae89065887f7e21c46af47bbb046edc46cdf2b663af9a2c1ae34034851f
-size 1870438
+oid sha256:766ec6b6148fb723d84938852ba043599e3a42929d6fa53362b707e636653d14
+size 1877877

models/tokenizer/as_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd471a819995e56dd4aefb6525b4fb7cb54d7cb5e996fa7e85acdd00b8586c74
-size 429491
+oid sha256:6a87b039391762a6c6ff78ebe8fd564dc70f94488d6ba3c9de287f4e128321f6
+size 429632

models/vocabulary/as_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55714b8ff99e197cb70a99feadc4f2eb8b928c63ec225a2ec5cce487f9b36d0d
-size 3925398
+oid sha256:635e37d5ab63d938c31e94632665a224be559880e7dad16b503669e16b038cb6
+size 4032114

models/vocabulary/as_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "as",
-  "vocabulary_size": 219027,
+  "vocabulary_size": 225407,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.059760376668044714,
+    "type_token_ratio": 0.05897458564719939,
     "coverage": {
-      "top_100": 0.2446919080599598,
-      "top_1000": 0.48975445539765006,
-      "top_5000": 0.6928702663989404,
-      "top_10000": 0.7681790167555828
+      "top_100": 0.24564961035969066,
+      "top_1000": 0.49087521062543366,
+      "top_5000": 0.6936500818391987,
+      "top_10000": 0.7687838775439132
     },
-    "hapax_count": 314664,
-    "hapax_ratio": 0.5895995997684053,
-    "total_documents": 19816
+    "hapax_count": 324963,
+    "hapax_ratio": 0.5904446099896433,
+    "total_documents": 20709
   }
 }

models/word_markov/as_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dee794f4598493fa43d8881826c834158cd1924e7bf01f8b09883966c2c4551
-size 51785769
+oid sha256:367f01f25fd917cfda125b68731c819ec87687b5de4c372f3c1438dab7fba73e
+size 53316862

models/word_markov/as_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "as",
-  "unique_contexts": 533621,
-  "total_transitions": 8910700
+  "unique_contexts": 550295,
+  "total_transitions": 9311616
 }

models/word_markov/as_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e48a73c55b3245526277a461699b91064b48048043751385238d1834df5c660
-size 152841623
+oid sha256:4df2480371e314f0e91998adaa2c4aaed48925dabab8264af3ab126aa903b609
+size 159459542

models/word_markov/as_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "as",
-  "unique_contexts": 4157114,
-  "total_transitions": 8890884
+  "unique_contexts": 4311912,
+  "total_transitions": 9290907
 }