Merge pull request #5336 from solgenomics/topic/fix_decode_utf8

Fixing decode latin characters
solgenomics · Feb 24, 2025 · 555122b · 555122b
2 parents fdaa340 + 289427b
commit 555122b
Showing 1 changed file with 46 additions and 29 deletions.
diff --git a/lib/SGN/Controller/AJAX/Audit.pm b/lib/SGN/Controller/AJAX/Audit.pm
@@ -15,6 +15,7 @@ use CXGN::Dataset::File;
 use CXGN::Tools::Run;
 use Cwd qw(cwd);
 use JSON;
+use Encode;
 
 BEGIN { extends 'Catalyst::Controller::REST' };
 
@@ -37,13 +38,18 @@ sub retrieve_results : Path('/ajax/audit/retrieve_results'){
     my $counter = 0;
 
     while (my ($audit_ts, $operation, $username, $logged_in_user, $before, $after, $transactioncode, $primary_key, $is_undo) = $h->fetchrow_array) {
+        # Ensure UTF-8 encoding for text fields to prevent encoding issues
+        $before = defined $before ? Encode::decode('UTF-8', $before, Encode::FB_DEFAULT) : "";
+        $after  = defined $after  ? Encode::decode('UTF-8', $after, Encode::FB_DEFAULT)  : "";
+
         $all_audits[$counter] = [$audit_ts, $operation, $username, $logged_in_user, $before, $after, $transactioncode, $primary_key, $is_undo];
         $counter++;
-        };
+    }
 
+    # Encode result properly before sending to response
+    my $json_string = encode_json(\@all_audits);
+    utf8::encode($json_string); # Convert to UTF-8 bytes
 
-    my $json_string;
-    $json_string = encode_json(\@all_audits);
     $c->stash->{rest} = {
         result => $json_string,
         };
@@ -57,11 +63,14 @@ sub retrieve_table_names : Path('/ajax/audit/retrieve_table_names'){
     $h->execute();
     my @ids;
     while (my ($drop_options) = $h->fetchrow_array) {
-        push @ids, $drop_options;
+        # Ensure UTF-8 decoding
+        push @ids, Encode::decode('UTF-8', $drop_options, Encode::FB_DEFAULT);
+    }
+
+    # Encode result properly before sending to response
+    my $json_string = encode_json(\@ids);
+    utf8::encode($json_string); # Convert to UTF-8 bytes
 
-    };
-    my $json_string;
-    $json_string = encode_json(\@ids);
     $c->stash->{rest} = {
         result1 => $json_string,
         };
@@ -88,22 +97,24 @@ sub retrieve_stock_audits : Path('/ajax/audit/retrieve_stock_audits'){
 
 
     my @matches;
-    for (my $i = 0; $i<$counter; $i++){
+    for (my $i = 0; $i < $counter; $i++) {
         my $operation = $all_audits[$i][1];
         my $stock_json_string;
-	eval {
-            if ($operation eq "DELETE"){
-                $stock_json_string = decode_json($before[$i]);
-            } else {
-                $stock_json_string = decode_json($after[$i]);
-            }
-	};
-	if ($@) {
-	    warn "Failed to decode JSON at index $i: $@";
+
+        eval {
+            my $json_text = ($operation eq "DELETE") ? $before[$i] : $after[$i];
+
+            # Convert Perl Unicode string to UTF-8 encoded bytes before decoding JSON
+            $json_text = Encode::encode('UTF-8', $json_text);
+            $stock_json_string = decode_json($json_text);
+        };
+        if ($@) {
+            warn "Failed to decode JSON at index $i: $@";
             next; # Skip this iteration in case of error
         }
+
         my $desired_uniquename = $stock_json_string->{'uniquename'};
-        if($stock_uniquename eq $desired_uniquename){
+        if ($stock_uniquename eq $desired_uniquename) {
             push @matches, $all_audits[$i];
         }
     }
@@ -140,27 +151,33 @@ sub retrieve_trial_audits : Path('/ajax/audit/retrieve_trial_audits'){
     my @matches;
     my $num_matches = 0; #this is to make sure only matched audits go into the matches array
 
-    for (my $i = 0; $i<$counter; $i++){
+    for (my $i = 0; $i < $counter; $i++) {
         my $operation = $all_audits[$i][1];
         my $json_string;
-        if($operation eq "DELETE"){
-            $json_string = decode_json($before[$i]);
-        }else{
-            $json_string = decode_json($after[$i]);
+
+        eval {
+            my $json_text = ($operation eq "DELETE") ? $before[$i] : $after[$i];
+
+            # Convert Perl Unicode string to UTF-8 encoded bytes before decoding JSON
+            $json_text = Encode::encode('UTF-8', $json_text);
+            $json_string = decode_json($json_text);
+        };
+        if ($@) {
+            warn "Failed to decode JSON at index $i: $@";
+            next; # Skip this iteration in case of error
         }
+
         my $desired_trial_id = $json_string->{'project_id'};
 
-        if($trial_id eq $desired_trial_id){
-
-
+        if ($trial_id eq $desired_trial_id) {
             $matches[$num_matches] = $all_audits[$i];
             $num_matches++;
-
         }
     }
 
-    my $match_trial_json;
-    $match_trial_json = encode_json(\@matches);
+    # Encode result properly before sending to response
+    my $match_trial_json = encode_json(\@matches);
+    utf8::encode($match_trial_json); # Convert to UTF-8 bytes
 
     $c->stash->{rest} = {
         match_project => $match_trial_json,