Upload postprocessing.py with huggingface_hub
Browse files- postprocessing.py +3 -2
postprocessing.py
CHANGED
|
@@ -68,8 +68,9 @@ def postprocess_text(text, cls = 'Text', text_format='markdown', table_format='l
|
|
| 68 |
|
| 69 |
def remove_nemotron_formatting(text):
|
| 70 |
text = text.replace('<tbc>', '')
|
| 71 |
-
|
| 72 |
-
|
|
|
|
| 73 |
|
| 74 |
def convert_mmd_to_plain_text_ours(mmd_text):
|
| 75 |
mmd_text = re.sub(r'<sup>(.*?)</sup>', r'^{\\1}', mmd_text, flags=re.DOTALL)
|
|
|
|
| 68 |
|
| 69 |
def remove_nemotron_formatting(text):
|
| 70 |
text = text.replace('<tbc>', '')
|
| 71 |
+
text = mmd_text.replace('\\<|unk|\\>', '')
|
| 72 |
+
text = mmd_text.replace('\\unknown', '')
|
| 73 |
+
return text
|
| 74 |
|
| 75 |
def convert_mmd_to_plain_text_ours(mmd_text):
|
| 76 |
mmd_text = re.sub(r'<sup>(.*?)</sup>', r'^{\\1}', mmd_text, flags=re.DOTALL)
|