diff --git a/extensions/removerepeatphrase.cpp b/extensions/removerepeatphrase.cpp index 0b16c41..d298244 100644 --- a/extensions/removerepeatphrase.cpp +++ b/extensions/removerepeatphrase.cpp @@ -2,6 +2,44 @@ constexpr wchar_t ERASED = 0xe012; // inside Unicode private use area +std::vector GenerateSuffixArray(const std::wstring& text) +{ + std::vector identity(text.size()); + for (int i = 0; i < text.size(); ++i) identity[i] = i; + std::vector suffixArray = identity; + // The below code is a more efficient way of doing this: + // std::sort(suffixArray.begin(), suffixArray.end(), [&](int a, int b) { return wcscmp(text.c_str() + a, text.c_str() + b) > 0; }); + std::stable_sort(suffixArray.begin(), suffixArray.end(), [&](int a, int b) { return text[a] > text[b]; }); + std::vector classes(text.begin(), text.end()); + for (int length = 1; length < text.size(); length *= 2) + { + // Determine equivalence class up to length, by checking length/2 equivalence of suffixes and their following length/2 suffixes + std::vector oldClasses = classes; + classes[suffixArray[0]] = 0; + for (int i = 1; i < text.size(); ++i) + { + int currentSuffix = suffixArray[i]; + int lastSuffix = suffixArray[i - 1]; + if (currentSuffix + length < text.size() && oldClasses[currentSuffix] == oldClasses[lastSuffix] && + oldClasses[currentSuffix + length / 2] == oldClasses.at(lastSuffix + length / 2)) // not completely certain that this will stay in range + classes[currentSuffix] = classes[lastSuffix]; + else classes[currentSuffix] = i; + } + + // Sort within equivalence class based on order of following suffix after length + // Orders up to length*2 + std::vector count = identity; + for (auto suffix : std::vector(suffixArray)) + { + int precedingSuffix = suffix - length; + if (precedingSuffix >= 0) suffixArray[count[classes[precedingSuffix]]++] = precedingSuffix; + } + } + for (int i = 0; i + 1 < text.size(); ++i) + assert(wcscmp(text.c_str() + suffixArray[i], text.c_str() + suffixArray[i + 1]) > 0); + return suffixArray; +} + bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) { if (sentenceInfo["text number"] == 0) return false; @@ -9,9 +47,7 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo) // This algorithm looks for repeating substrings (in other words, common prefixes among the set of suffixes) of the sentence with length > 6 // It then looks for any regions of characters at least twice as long as the substring made up only of characters in the substring, and erases them // If this results in the common prefix being completely erased from the string, the common prefix is copied to the last location where it was located in the original string - std::vector suffixArray(sentence.size()); - for (int i = 0; i < sentence.size(); ++i) suffixArray[i] = i; - std::sort(suffixArray.begin(), suffixArray.end(), [&](int a, int b) { return wcsncmp(sentence.c_str() + a, sentence.c_str() + b, 5000) > 0; }); + std::vector suffixArray = GenerateSuffixArray(sentence); for (int i = 0; i + 1 < sentence.size(); ++i) { int commonPrefixLength = 0;