Update tokenizer.py
Browse files- tokenizer.py +2 -1
tokenizer.py
CHANGED
|
@@ -20,7 +20,8 @@ class MedicalcodeTokenizer(PreTrainedTokenizer):
|
|
| 20 |
|
| 21 |
def _tokenize(self, text):
|
| 22 |
if text in self.code2tok:
|
| 23 |
-
return self.code2tok[text]
|
|
|
|
| 24 |
return self._infer_and_register(text)
|
| 25 |
|
| 26 |
def _convert_token_to_id(self, token):
|
|
|
|
| 20 |
|
| 21 |
def _tokenize(self, text):
|
| 22 |
if text in self.code2tok:
|
| 23 |
+
return [f"token_{t}" for t in self.code2tok[text]]
|
| 24 |
+
return self._infer_and_register(text)
|
| 25 |
return self._infer_and_register(text)
|
| 26 |
|
| 27 |
def _convert_token_to_id(self, token):
|