Fix incorrect UTF-8 -> 1252 conversion when using liblcf without ICU

src/encoder.cpp

@@ -183,9 +183,10 @@ void Encoder::Convert(std::string& str, int conv_dst, int) {
 		// From UTF-8 to 1252
 		// Based on https://stackoverflow.com/q/23689733/
 		_buffer.resize(str.size() + 1);
-		uint32_t codepoint;
+		uint32_t codepoint = 0;
 
-		for (size_t str_idx = 0; str_idx < str.size(); ++str_idx) {
+		size_t str_idx = 0;
+		while (str_idx < str.size()) {
 			unsigned char ch = str[str_idx];
 			if (ch <= 0x7F) {
 				codepoint = ch;
@@ -199,15 +200,20 @@ void Encoder::Convert(std::string& str, int conv_dst, int) {
 				codepoint = ch & 0x07;
 			}
 			++str_idx;
-			ch = str[str_idx];
+			if (str_idx < str.size()) {
+				ch = str[str_idx];
+			} else {
+				ch = '\0';
+			}
 			if (((ch & 0xC0) != 0x80) && (codepoint <= 0x10ffff)) {
 				if (codepoint <= 255) {
 					_buffer[buf_idx] = static_cast<char>(codepoint);
 				} else {
 					_buffer[buf_idx] = '?';
 				}
+
+				++buf_idx;
 			}
-			++buf_idx;
 		}
 	}
 

tests/encoder.cpp

@@ -0,0 +1,61 @@
+/*
+ * This file is part of liblcf. Copyright (c) liblcf authors.
+ * https://github.com/EasyRPG/liblcf - https://easyrpg.org
+ *
+ * liblcf is Free/Libre Open Source Software, released under the MIT License.
+ * For the full copyright and license information, please view the COPYING
+ * file that was distributed with this source code.
+ */
+
+#include "doctest.h"
+#include "lcf/config.h"
+#include "lcf/reader_util.h"
+#include "lcf/encoder.h"
+
+TEST_SUITE_BEGIN("encoder");
+
+TEST_CASE("1252 (Western Europe)") {
+	lcf::Encoder enc("1252");
+	std::string orig = "\xBFH\xEBll\xF6\xA1"; // ¿Hëllö¡ in Latin-1
+	std::string text = orig;
+	enc.Encode(text);
+	REQUIRE_EQ(text, "¿Hëllö¡");
+
+	enc.Decode(text);
+	REQUIRE_EQ(text, orig);
+}
+
+TEST_CASE("windows-1252 (Alias / Western Europe)") {
+	lcf::Encoder enc("windows-1252");
+	std::string orig = "\xBFH\xEBll\xF6\xA1"; // ¿Hëllö¡ in Latin-1
+	std::string text = orig;
+	enc.Encode(text);
+	REQUIRE_EQ(text, "¿Hëllö¡");
+
+	enc.Decode(text);
+	REQUIRE_EQ(text, orig);
+}
+
+TEST_CASE("932 (Japanese)" * doctest::skip(LCF_SUPPORT_ICU == 0)) {
+	lcf::Encoder enc("932");
+	// こんにちは in Shift-JIS
+	std::string orig = "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD";
+	std::string text = orig;
+	enc.Encode(text);
+	REQUIRE_EQ(text, "こんにちは");
+
+	enc.Decode(text);
+	REQUIRE_EQ(text, orig);
+}
+
+TEST_CASE("Normalize" * doctest::skip(LCF_SUPPORT_ICU == 0)) {
+	REQUIRE_EQ(lcf::ReaderUtil::Normalize("ä①ÄБ"), "ä1äб");
+
+	// The Å is precomposed, the å is precomposed
+	REQUIRE_EQ(lcf::ReaderUtil::Normalize("Å"), "å");
+
+	// The Å is A + ◌̊, the å is precomposed
+	REQUIRE_EQ(lcf::ReaderUtil::Normalize("Å"), "å");
+}
+
+TEST_SUITE_END();