初步实现对utf-8文本的支持

Author: undefined-ux

.gitignore

@@ -3,3 +3,4 @@ x64/
 cmake-build-debug/
 *.pdb.idea/
 cmake-build-release/
+cmake-build-debug-mingw/

.idea/editor.xml

@@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.27)
 
 PROJECT(JsonParser C)
 set(CMAKE_C_STANDARD 17)
-
 add_library(jsonParserLib
         core/Json.h
         core/parser/parser.h
@@ -11,15 +10,16 @@ add_library(jsonParserLib
         core/types/JsonObject.h
         core/types/JsonString.h
         core/utils/outputer.h
+        core/utils/outputer.c
         core/parser/parser.c
         core/types/JsonValue.c
         core/types/JsonArray.c
         core/types/JsonObject.c
         core/types/JsonString.c
-        core/utils/outputer.c
-        core/utils/UTF2GBK.cpp
-        core/utils/UTF2GBK.h
+        core/utils/utf2gbk/UTF2GBK.c
+        core/utils/utf2gbk/UTF2GBK.h
 )
+target_link_libraries(jsonParserLib PRIVATE iconv)
 
 add_executable(json
         cli/main.c

.idea/encodings.xml

@@ -12,6 +12,10 @@ struct CommandLineArgs {
 	FILE* output;        // 输出流
 	int compress;        // 是否压缩
 	int format;          // 是否格式化
+    int utf8Text;         // 是否为utf-8 文本 是则需资源回收时删除中间文件
+    char* convertCacheFilePath; // 为utf-8文本时转换为gbk格式时生成的临时文件
+    char* outputFilePath; // 需要输出文件路径， 当为utf-8文本时用于转换回utf-8文本
+
 };
 
 // 函数声明
@@ -40,6 +44,9 @@ int main(const int argc, char* argv[]) {
     }
     if (args.output != stdout) {
         fclose(args.output);
+        if(args.utf8Text) {
+            convertGbkToUtf8(args.output);
+        }
     }
 	return 0;
 }
@@ -53,7 +60,8 @@ struct CommandLineArgs parseCommandLineArgs(int argc, char* argv[]) {
     args.output = stdout;
     args.compress = 0;
     args.format = 1;
-
+    args.utf8Text = 0;
+    args.convertCacheFilePath = "";
     // 标记是否已经出现了--format或--compress
     int formatSeen = 0;
     int compressSeen = 0;
@@ -64,6 +72,7 @@ struct CommandLineArgs parseCommandLineArgs(int argc, char* argv[]) {
             if (i + 1 < argc) {
                 printf("Output: %s\n", argv[i + 1]);
                 args.output = fopen(argv[i + 1], "w");
+                args.outputFilePath = argv[i+1];
                 if (args.output == NULL) {
                     perror("Error opening output file");
                     exit(EXIT_FAILURE);
@@ -78,7 +87,15 @@ struct CommandLineArgs parseCommandLineArgs(int argc, char* argv[]) {
         else if (strcmp(argv[i], "--input") == 0 || strcmp(argv[i], "-if") == 0) {
             // 指定输入流
             if (i + 1 < argc) {
-                args.input = fopen(argv[i + 1], "r");
+                FILE* f = fopen(argv[i + 1], "r");
+                args.convertCacheFilePath = "__cache.json";
+                if(isUtf8(f)) {
+                    printf("INFO: Is UTF-8 Text\n");
+                    args.input = convertUtf8ToGbk(f,args.convertCacheFilePath);
+                    args.utf8Text = 1;
+                }else {
+                    args.input = fopen(argv[i + 1], "r");
+                }
                 if (args.input == NULL) {
                     perror("Error opening input file");
                     exit(EXIT_FAILURE);

CMakeLists.txt

@@ -4,4 +4,5 @@
 #include "types/JsonValue.h"
 #include "types/JsonArray.h"
 #include "utils/outputer.h"
-#include "./parser/parser.h"
+#include "parser/parser.h"
+#include "utils/utf2gbk/UTF2GBK.h"

cli/main.c

@@ -0,0 +1,134 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <iconv.h>
+
+FILE* convertUtf8ToGbk(FILE* input, const char* outputFileName) {
+    iconv_t cd;
+    cd = iconv_open("GBK", "UTF-8");
+    if (cd == (iconv_t)-1) {
+        perror("iconv_open");
+        exit(EXIT_FAILURE);
+    }
+
+    FILE* output = fopen(outputFileName, "w");
+    if (output == NULL) {
+        perror("fopen");
+        exit(EXIT_FAILURE);
+    }
+
+    char inputBuffer[4096];
+    char outputBuffer[8192];
+    size_t bytesRead, inputBytesLeft, outputBytesLeft;
+
+    while ((bytesRead = fread(inputBuffer, 1, sizeof(inputBuffer), input)) > 0) {
+        inputBytesLeft = bytesRead;
+        char* inputPtr = inputBuffer;
+
+        outputBytesLeft = sizeof(outputBuffer);
+        char* outputPtr = outputBuffer;
+
+        if (iconv(cd, &inputPtr, &inputBytesLeft, &outputPtr, &outputBytesLeft) == (size_t)-1) {
+            perror("iconv");
+            exit(EXIT_FAILURE);
+        }
+
+        size_t outputBytesWritten = sizeof(outputBuffer) - outputBytesLeft;
+        fwrite(outputBuffer, 1, outputBytesWritten, output);
+    }
+
+    iconv_close(cd);
+    fclose(input);
+    fclose(output);
+
+    return fopen(outputFileName, "r");
+}
+
+void convertGbkToUtf8(FILE* file) {
+    // Get the size of the file
+    fseek(file, 0, SEEK_END);
+    long size = ftell(file);
+    rewind(file);
+
+    // Read the content into a buffer
+    char* buffer = (char*)malloc(size + 1);
+    if (buffer == NULL) {
+        perror("malloc");
+        exit(EXIT_FAILURE);
+    }
+
+    size_t bytesRead = fread(buffer, 1, size, file);
+    buffer[bytesRead] = '\0'; // Null-terminate the buffer
+
+    // Close the original file
+    fclose(file);
+
+    // Convert the content from GBK to UTF-8
+    iconv_t cd;
+    cd = iconv_open("UTF-8", "GBK");
+    if (cd == (iconv_t)-1) {
+        perror("iconv_open");
+        exit(EXIT_FAILURE);
+    }
+
+    // Create a temporary buffer for the converted content
+    char* tempBuffer = (char*)malloc(size * 4); // UTF-8 can be up to 4 bytes per character
+    if (tempBuffer == NULL) {
+        perror("malloc");
+        exit(EXIT_FAILURE);
+    }
+
+    char* inputPtr = buffer;
+    char* outputPtr = tempBuffer;
+    size_t inputBytesLeft = bytesRead;
+    size_t outputBytesLeft = size * 4;
+
+    if (iconv(cd, &inputPtr, &inputBytesLeft, &outputPtr, &outputBytesLeft) == (size_t)-1) {
+        perror("iconv");
+        exit(EXIT_FAILURE);
+    }
+
+    // Close the iconv descriptor
+    iconv_close(cd);
+
+    // Reopen the original file for writing
+    file = fopen("converted_file.txt", "w");
+    if (file == NULL) {
+        perror("fopen");
+        exit(EXIT_FAILURE);
+    }
+
+    // Write the converted content back to the file
+    fwrite(tempBuffer, 1, size * 4 - outputBytesLeft, file);
+
+    // Close the file and free the buffers
+    fclose(file);
+    free(buffer);
+    free(tempBuffer);
+}
+
+
+int isUtf8(FILE* file) {
+    rewind(file); // 将文件指针定位到文件开头
+
+    // 读取文件的前三个字节
+    char bom[3];
+    size_t bytesRead = fread(bom, 1, 3, file);
+
+    // 如果文件小于3个字节，返回0
+    if (bytesRead < 3) {
+        rewind(file);
+        return 0;
+    }
+
+    // 判断是否为UTF-8 without BOM编码
+    if (bom[0] == (char)0xEF && bom[1] == (char)0xBB && bom[2] == (char)0xBF) {
+        // 文件包含BOM，不是UTF-8 without BOM编码
+        rewind(file);
+        return 0;
+    } else {
+        // 文件不包含BOM，可能是UTF-8 without BOM编码
+        rewind(file);
+        return 1;
+    }
+}

core/Json.h

@@ -0,0 +1,9 @@
+//
+// Created by undefined on 2023/12/10.
+//
+#pragma once
+#include <stdio.h>
+
+FILE* convertUtf8ToGbk(FILE* input, const char* outputFileName);
+void convertGbkToUtf8(FILE* input);
+int isUtf8(FILE* file);