Upload postprocessing.py with huggingface_hub
Browse files- postprocessing.py +3 -3
postprocessing.py
CHANGED
|
@@ -68,9 +68,9 @@ def postprocess_text(text, cls = 'Text', text_format='markdown', table_format='l
|
|
| 68 |
|
| 69 |
def remove_nemotron_formatting(text):
|
| 70 |
text = text.replace('<tbc>', '')
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
def convert_mmd_to_plain_text_ours(mmd_text):
|
| 75 |
mmd_text = re.sub(r'<sup>(.*?)</sup>', r'^{\\1}', mmd_text, flags=re.DOTALL)
|
| 76 |
mmd_text = re.sub(r'<sub>(.*?)</sub>', r'_{\\1}', mmd_text, flags=re.DOTALL)
|
|
|
|
| 68 |
|
| 69 |
def remove_nemotron_formatting(text):
|
| 70 |
text = text.replace('<tbc>', '')
|
| 71 |
+
text = text.replace('\\<|unk|\\>', '')
|
| 72 |
+
text = text.replace('\\unknown', '')
|
| 73 |
+
return text
|
| 74 |
def convert_mmd_to_plain_text_ours(mmd_text):
|
| 75 |
mmd_text = re.sub(r'<sup>(.*?)</sup>', r'^{\\1}', mmd_text, flags=re.DOTALL)
|
| 76 |
mmd_text = re.sub(r'<sub>(.*?)</sub>', r'_{\\1}', mmd_text, flags=re.DOTALL)
|