allow different codepages

GUI/host/host.cc

@@ -31,14 +31,13 @@ namespace
 
 	void DispatchText(ThreadParam tp, const BYTE* text, int len)
 	{
-		if (!text || len <= 0) return;
 		LOCK(hostMutex);
-		TextThread *it;
+		if (textThreadsByParams[tp] == nullptr)
-		if ((it = textThreadsByParams[tp]) == nullptr)
+		{
-			if (textThreadsByParams.size() < MAX_THREAD_COUNT)
+			if (textThreadsByParams.size() > MAX_THREAD_COUNT) return Host::AddConsoleOutput(L"too many text threads: can't create more");
-				OnCreate(it = textThreadsByParams[tp] = new TextThread(tp, Host::GetHookParam(tp).type));
+			OnCreate(textThreadsByParams[tp] = new TextThread(tp));
-			else return Host::AddConsoleOutput(L"too many text threads: stopping");
+		}
-		it->AddText(text, len);
+		textThreadsByParams[tp]->AddText(text, len);
 	}
 
 	void RemoveThreads(std::function<bool(ThreadParam)> removeIf)
@@ -144,7 +143,7 @@ namespace Host
 	void Start(ProcessEventCallback onAttach, ProcessEventCallback onDetach, ThreadEventCallback onCreate, ThreadEventCallback onRemove, TextThread::OutputCallback output)
 	{
 		OnAttach = onAttach; OnDetach = onDetach; OnCreate = onCreate; OnRemove = onRemove; TextThread::Output = output;
-		OnCreate(textThreadsByParams[CONSOLE] = new TextThread(CONSOLE, USING_UNICODE));
+		OnCreate(textThreadsByParams[CONSOLE] = new TextThread(CONSOLE));
 		StartPipe();
 	}
 

GUI/host/textthread.cc

@@ -8,7 +8,7 @@
 #include <regex>
 #include <algorithm>
 
-TextThread::TextThread(ThreadParam tp, DWORD status) : handle(threadCounter++), name(Host::GetHookName(tp.pid, tp.hook)), tp(tp), status(status) {}
+TextThread::TextThread(ThreadParam tp) : handle(threadCounter++), name(Host::GetHookName(tp.pid, tp.hook)), tp(tp), hp(Host::GetHookParam(tp)) {}
 
 TextThread::~TextThread()
 {
@@ -19,7 +19,7 @@ TextThread::~TextThread()
 
 std::wstring TextThread::GetStorage()
 {
-	LOCK(ttMutex);
+	LOCK(threadMutex);
 	return storage;
 }
 
@@ -27,7 +27,7 @@ void TextThread::Flush()
 {
 	std::wstring sentence;
 	{
-		LOCK(ttMutex);
+		LOCK(threadMutex);
 		if (buffer.empty()) return;
 		if (buffer.size() < maxBufferSize && GetTickCount() - timestamp < flushDelay) return;
 		sentence = buffer;
@@ -46,17 +46,18 @@ void TextThread::AddSentence(std::wstring sentence)
 	// Dispatch to extensions occurs here. Don't hold mutex! Extensions might take a while!
 	if (Output(this, sentence))
 	{
-		LOCK(ttMutex);
+		LOCK(threadMutex);
 		storage += sentence;
 	}
 }
 
 void TextThread::AddText(const BYTE* data, int len)
 {
-	LOCK(ttMutex);
+	if (len < 0) return;
-	buffer += status & USING_UNICODE
+	LOCK(threadMutex);
+	buffer += hp.type & USING_UNICODE
 		? std::wstring((wchar_t*)data, len / 2)
-		: StringToWideString(std::string((char*)data, len), status & USING_UTF8 ? CP_UTF8 : SHIFT_JIS);
+		: StringToWideString(std::string((char*)data, len), hp.codepage);
 	if (std::all_of(buffer.begin(), buffer.end(), [&](wchar_t c) { return repeatingChars.count(c) > 0; })) buffer.clear();
 	timestamp = GetTickCount();
 }

GUI/host/textthread.h

@@ -19,7 +19,7 @@ public:
 	inline static int maxBufferSize = 200;
 	inline static int threadCounter = 0;
 
-	TextThread(ThreadParam tp, DWORD status);
+	TextThread(ThreadParam tp);
 	~TextThread();
 
 	std::wstring GetStorage();
@@ -29,6 +29,7 @@ public:
 	const int64_t handle;
 	const std::wstring name;
 	const ThreadParam tp;
+	const HookParam hp;
 
 private:
 	void Flush();
@@ -36,8 +37,7 @@ private:
 	std::wstring buffer;
 	std::wstring storage;
 	std::unordered_set<wchar_t> repeatingChars;
-	std::recursive_mutex ttMutex;
+	std::recursive_mutex threadMutex;
-	DWORD status;
 
 	HANDLE deletionEvent = CreateEventW(nullptr, FALSE, FALSE, NULL);
 	std::thread flushThread = std::thread([&] { while (WaitForSingleObject(deletionEvent, 10) == WAIT_TIMEOUT) Flush(); });

GUI/misc.cpp

@@ -53,6 +53,14 @@ namespace
 		}
 		RCode.remove(0, 1);
 
+		// [codepage#]
+		QRegularExpressionMatch codepage = QRegularExpression("^([0-9]+)#").match(RCode);
+		if (codepage.hasMatch())
+		{
+			hp.codepage = codepage.captured(1).toInt();
+			RCode.remove(0, codepage.captured(0).length());
+		}
+
 		// [*deref_offset|0]
 		if (RCode.at(0).unicode() == L'0') RCode.remove(0, 1); // Legacy
 		QRegularExpressionMatch deref = QRegularExpression("^\\*(\\-?[[:xdigit:]]+)").match(RCode);
@@ -109,6 +117,14 @@ namespace
 			HCode.remove(0, 1);
 		}
 
+		// [codepage#]
+		QRegularExpressionMatch codepage = QRegularExpression("^([0-9]+)#").match(HCode);
+		if (codepage.hasMatch())
+		{
+			hp.codepage = codepage.captured(1).toInt();
+			HCode.remove(0, codepage.captured(0).length());
+		}
+
 		// data_offset
 		QRegularExpressionMatch dataOffset = QRegularExpression("^\\-?[[:xdigit:]]+").match(HCode);
 		if (!dataOffset.hasMatch()) return {};
@@ -164,12 +180,31 @@ namespace
 		return hp;
 	}
 
+	QString GenerateRCode(HookParam hp)
+	{
+		QString RCode = "/R";
+		QTextStream codeBuilder(&RCode);
+
+		if (hp.type & USING_UNICODE) codeBuilder << "Q";
+		else if (hp.type & USING_UTF8) codeBuilder << "V";
+		else codeBuilder << "S";
+
+		if (hp.codepage != SHIFT_JIS && hp.codepage != CP_UTF8) codeBuilder << hp.codepage << "#";
+
+		codeBuilder.setIntegerBase(16);
+		codeBuilder.setNumberFlags(QTextStream::UppercaseDigits);
+
+		if (hp.type & DATA_INDIRECT) codeBuilder << "*" << hp.index;
+
+		codeBuilder << "@" << hp.address;
+
+		return RCode;
+	}
+
 	QString GenerateHCode(HookParam hp, DWORD processId)
 	{
 		QString HCode = "/H";
 		QTextStream codeBuilder(&HCode);
-		codeBuilder.setIntegerBase(16);
-		codeBuilder.setNumberFlags(QTextStream::UppercaseDigits);
 
 		if (hp.type & USING_UNICODE)
 		{
@@ -185,6 +220,11 @@ namespace
 		}
 		if (hp.type & NO_CONTEXT) codeBuilder << "N";
 
+		if (hp.codepage != SHIFT_JIS && hp.codepage != CP_UTF8) codeBuilder << hp.codepage << "#";
+
+		codeBuilder.setIntegerBase(16);
+		codeBuilder.setNumberFlags(QTextStream::UppercaseDigits);
+
 		if (hp.offset < 0) hp.offset += 4;
 		if (hp.split < 0) hp.split += 4;
 
@@ -214,24 +254,6 @@ namespace
 
 		return HCode;
 	}
-
-	QString GenerateRCode(HookParam hp)
-	{
-		QString RCode = "/R";
-		QTextStream codeBuilder(&RCode);
-		codeBuilder.setIntegerBase(16);
-		codeBuilder.setNumberFlags(QTextStream::UppercaseDigits);
-
-		if (hp.type & USING_UNICODE) codeBuilder << "Q";
-		else if (hp.type & USING_UTF8) codeBuilder << "V";
-		else codeBuilder << "S";
-
-		if (hp.type & DATA_INDIRECT) codeBuilder << "*" << hp.index;
-
-		codeBuilder << "@" << hp.address;
-
-		return RCode;
-	}
 }
 
 std::optional<HookParam> ParseCode(QString code)

GUI/misc.h

@@ -13,11 +13,11 @@ QString GenerateCode(HookParam hp, DWORD processId);
 
 static QString CodeInfoDump =
 "Enter hook code\r\n\
-/H{A|B|W|S|Q|V}[N]data_offset[*deref_offset1][:split_offset[*deref_offset2]]@addr[:module[:func]]\r\n\
+/H{A|B|W|S|Q|V}[N][codepage#]data_offset[*deref_offset1][:split_offset[*deref_offset2]]@addr[:module[:func]]\r\n\
 OR\r\n\
 Enter read code\r\n\
-/R{S|Q|V}[*deref_offset|0]@addr\r\n\
+/R{S|Q|V}[#codepage][*deref_offset|0]@addr\r\n\
-All numbers in hexadecimal\r\n\
+All numbers except codepage in hexadecimal\r\n\
 A/B: Shift-JIS char little/big endian\r\n\
 W: UTF-16 char\r\n\
 S/Q/V: Shift-JIS/UTF-16/UTF-8 string\r\n\

GUI/tests.cpp

@@ -3,7 +3,7 @@
 
 static int TESTS = []
 {
-	assert(ParseCode("/HQ-c*C:C*1C@4AA:gdi.dll:GetTextOutA"));
+	assert(ParseCode("/HQN936#-c*C:C*1C@4AA:gdi.dll:GetTextOutA"));
 	assert(ParseCode("/HB4@0"));
 	assert(ParseCode("/RS*10@44"));
 	assert(!ParseCode("HQ@4"));

include/types.h

@@ -20,6 +20,7 @@ struct HookParam
 	wchar_t module[MAX_MODULE_SIZE];
 	char function[MAX_MODULE_SIZE];
 	DWORD type; // flags
+	UINT codepage; // text encoding
 	WORD length_offset; // index of the string length
 	DWORD user_value; // 7/20/2014: jichi additional parameters for PSP games
 

vnrhook/hijack/texthook.cc

@@ -232,13 +232,16 @@ bool TextHook::InsertHookCode()
 
 bool TextHook::UnsafeInsertHookCode()
 {
-	if (hp.type & MODULE_OFFSET)  // Map hook offset to real address.
+	if (hp.type & MODULE_OFFSET)  // Map hook offset to real address
 		if (hp.type & FUNCTION_OFFSET)
 			if (FARPROC function = GetProcAddress(GetModuleHandleW(hp.module), hp.function)) hp.insertion_address += (uint64_t)function;
 			else return ConsoleOutput("Textractor: UnsafeInsertHookCode: FAILED: function not present"), false;
 		else if (HMODULE moduleBase = GetModuleHandleW(hp.module)) hp.insertion_address += (uint64_t)moduleBase;
 		else return ConsoleOutput("Textractor: UnsafeInsertHookCode: FAILED: module not present"), false;
 
+	if (hp.type & USING_UTF8) hp.codepage = CP_UTF8;
+	if (hp.codepage == 0) hp.codepage = SHIFT_JIS; // Use Shift-JIS unless custom encoding was specified
+
 	BYTE* original;
 	insert:
 	if (MH_STATUS err = MH_CreateHook((void*)hp.insertion_address, (void*)trampoline, (void**)&original))