more efficient suffix array construction

extensions/removerepeatphrase.cpp

@@ -2,6 +2,44 @@
 
 constexpr wchar_t ERASED = 0xe012; // inside Unicode private use area
 
+std::vector<int> GenerateSuffixArray(const std::wstring& text)
+{
+	std::vector<int> identity(text.size());
+	for (int i = 0; i < text.size(); ++i) identity[i] = i;
+	std::vector<int> suffixArray = identity;
+	// The below code is a more efficient way of doing this:
+	// std::sort(suffixArray.begin(), suffixArray.end(), [&](int a, int b) { return wcscmp(text.c_str() + a, text.c_str() + b) > 0; });
+	std::stable_sort(suffixArray.begin(), suffixArray.end(), [&](int a, int b) { return text[a] > text[b]; });
+	std::vector<int> classes(text.begin(), text.end());
+	for (int length = 1; length < text.size(); length *= 2)
+	{
+		// Determine equivalence class up to length, by checking length/2 equivalence of suffixes and their following length/2 suffixes
+		std::vector<int> oldClasses = classes;
+		classes[suffixArray[0]] = 0;
+		for (int i = 1; i < text.size(); ++i)
+		{
+			int currentSuffix = suffixArray[i];
+			int lastSuffix = suffixArray[i - 1];
+			if (currentSuffix + length < text.size() && oldClasses[currentSuffix] == oldClasses[lastSuffix] &&
+				oldClasses[currentSuffix + length / 2] == oldClasses.at(lastSuffix + length / 2)) // not completely certain that this will stay in range
+				classes[currentSuffix] = classes[lastSuffix];
+			else classes[currentSuffix] = i;
+		}
+
+		// Sort within equivalence class based on order of following suffix after length
+		// Orders up to length*2
+		std::vector<int> count = identity;
+		for (auto suffix : std::vector(suffixArray))
+		{
+			int precedingSuffix = suffix - length;
+			if (precedingSuffix >= 0) suffixArray[count[classes[precedingSuffix]]++] = precedingSuffix;
+		}
+	}
+	for (int i = 0; i + 1 < text.size(); ++i)
+		assert(wcscmp(text.c_str() + suffixArray[i], text.c_str() + suffixArray[i + 1]) > 0);
+	return suffixArray;
+}
+
 bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
 {
 	if (sentenceInfo["text number"] == 0) return false;
@@ -9,9 +47,7 @@ bool ProcessSentence(std::wstring& sentence, SentenceInfo sentenceInfo)
 	// This algorithm looks for repeating substrings (in other words, common prefixes among the set of suffixes) of the sentence with length > 6
 	// It then looks for any regions of characters at least twice as long as the substring made up only of characters in the substring, and erases them
 	// If this results in the common prefix being completely erased from the string, the common prefix is copied to the last location where it was located in the original string
-	std::vector<int> suffixArray(sentence.size());
+	std::vector<int> suffixArray = GenerateSuffixArray(sentence);
-	for (int i = 0; i < sentence.size(); ++i) suffixArray[i] = i;
-	std::sort(suffixArray.begin(), suffixArray.end(), [&](int a, int b) { return wcsncmp(sentence.c_str() + a, sentence.c_str() + b, 5000) > 0; });
 	for (int i = 0; i + 1 < sentence.size(); ++i)
 	{
 		int commonPrefixLength = 0;