From f06244e1259f5ce7a1d40392792e15e6c30d8ed2 Mon Sep 17 00:00:00 2001 From: chaihahaha Date: Fri, 12 Jan 2024 22:43:43 +0800 Subject: [PATCH] Add EOS token at end of sentence to prevent repetition. (#490) Fix repetition at the end of sentence by adding a token id. --- LunaTranslator/LunaTranslator/translator/ort_sp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/LunaTranslator/LunaTranslator/translator/ort_sp.py b/LunaTranslator/LunaTranslator/translator/ort_sp.py index ae266260..6a1e017a 100644 --- a/LunaTranslator/LunaTranslator/translator/ort_sp.py +++ b/LunaTranslator/LunaTranslator/translator/ort_sp.py @@ -158,6 +158,7 @@ class TS(basetrans): ) input_ids_len = n_tokens.value input_ids_py = [token_ids[i] for i in range(input_ids_len)] + input_ids_py += [1] # add EOS token to notify the end of sentence and prevent repetition return input_ids_py def decode_from_ids(self, output_ids_py):