From c0c00ee841c68505312f5e68d81194f7b9032ffa Mon Sep 17 00:00:00 2001 From: Chris Simoes Date: Wed, 19 Feb 2025 19:49:04 -0500 Subject: [PATCH 1/2] adding decode json to avoid problems with latin characters --- lib/SGN/Controller/AJAX/Audit.pm | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/SGN/Controller/AJAX/Audit.pm b/lib/SGN/Controller/AJAX/Audit.pm index a25195392f..fb5d4710d8 100644 --- a/lib/SGN/Controller/AJAX/Audit.pm +++ b/lib/SGN/Controller/AJAX/Audit.pm @@ -15,6 +15,7 @@ use CXGN::Dataset::File; use CXGN::Tools::Run; use Cwd qw(cwd); use JSON; +use Encode; BEGIN { extends 'Catalyst::Controller::REST' }; @@ -60,8 +61,11 @@ sub retrieve_table_names : Path('/ajax/audit/retrieve_table_names'){ push @ids, $drop_options; }; - my $json_string; - $json_string = encode_json(\@ids); + + # Fixing problems with Latin characters + @ids = map { Encode::decode('UTF-8', $_, Encode::FB_DEFAULT) } @ids; + + my $json_string = encode_json(\@ids); $c->stash->{rest} = { result1 => $json_string, }; @@ -93,9 +97,9 @@ sub retrieve_stock_audits : Path('/ajax/audit/retrieve_stock_audits'){ my $stock_json_string; eval { if ($operation eq "DELETE"){ - $stock_json_string = decode_json($before[$i]); + $stock_json_string = decode_json(Encode::decode('UTF-8', $before[$i])); } else { - $stock_json_string = decode_json($after[$i]); + $stock_json_string = decode_json(Encode::decode('UTF-8', $after[$i])); } }; if ($@) { @@ -144,9 +148,9 @@ sub retrieve_trial_audits : Path('/ajax/audit/retrieve_trial_audits'){ my $operation = $all_audits[$i][1]; my $json_string; if($operation eq "DELETE"){ - $json_string = decode_json($before[$i]); + $json_string = decode_json(Encode::decode('UTF-8', $before[$i])); }else{ - $json_string = decode_json($after[$i]); + $json_string = decode_json(Encode::decode('UTF-8', $after[$i])); } my $desired_trial_id = $json_string->{'project_id'}; From 289427b0e305c9d350ea1f1810bca355a2d56658 Mon Sep 17 00:00:00 2001 From: Chris Simoes Date: Thu, 20 Feb 2025 08:51:42 -0500 Subject: [PATCH 2/2] adding more UTF8 checks --- lib/SGN/Controller/AJAX/Audit.pm | 75 +++++++++++++++++++------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/lib/SGN/Controller/AJAX/Audit.pm b/lib/SGN/Controller/AJAX/Audit.pm index fb5d4710d8..79d4de2a3c 100644 --- a/lib/SGN/Controller/AJAX/Audit.pm +++ b/lib/SGN/Controller/AJAX/Audit.pm @@ -38,13 +38,18 @@ sub retrieve_results : Path('/ajax/audit/retrieve_results'){ my $counter = 0; while (my ($audit_ts, $operation, $username, $logged_in_user, $before, $after, $transactioncode, $primary_key, $is_undo) = $h->fetchrow_array) { + # Ensure UTF-8 encoding for text fields to prevent encoding issues + $before = defined $before ? Encode::decode('UTF-8', $before, Encode::FB_DEFAULT) : ""; + $after = defined $after ? Encode::decode('UTF-8', $after, Encode::FB_DEFAULT) : ""; + $all_audits[$counter] = [$audit_ts, $operation, $username, $logged_in_user, $before, $after, $transactioncode, $primary_key, $is_undo]; $counter++; - }; + } + # Encode result properly before sending to response + my $json_string = encode_json(\@all_audits); + utf8::encode($json_string); # Convert to UTF-8 bytes - my $json_string; - $json_string = encode_json(\@all_audits); $c->stash->{rest} = { result => $json_string, }; @@ -58,14 +63,14 @@ sub retrieve_table_names : Path('/ajax/audit/retrieve_table_names'){ $h->execute(); my @ids; while (my ($drop_options) = $h->fetchrow_array) { - push @ids, $drop_options; - - }; - - # Fixing problems with Latin characters - @ids = map { Encode::decode('UTF-8', $_, Encode::FB_DEFAULT) } @ids; + # Ensure UTF-8 decoding + push @ids, Encode::decode('UTF-8', $drop_options, Encode::FB_DEFAULT); + } + # Encode result properly before sending to response my $json_string = encode_json(\@ids); + utf8::encode($json_string); # Convert to UTF-8 bytes + $c->stash->{rest} = { result1 => $json_string, }; @@ -92,22 +97,24 @@ sub retrieve_stock_audits : Path('/ajax/audit/retrieve_stock_audits'){ my @matches; - for (my $i = 0; $i<$counter; $i++){ + for (my $i = 0; $i < $counter; $i++) { my $operation = $all_audits[$i][1]; my $stock_json_string; - eval { - if ($operation eq "DELETE"){ - $stock_json_string = decode_json(Encode::decode('UTF-8', $before[$i])); - } else { - $stock_json_string = decode_json(Encode::decode('UTF-8', $after[$i])); - } - }; - if ($@) { - warn "Failed to decode JSON at index $i: $@"; + + eval { + my $json_text = ($operation eq "DELETE") ? $before[$i] : $after[$i]; + + # Convert Perl Unicode string to UTF-8 encoded bytes before decoding JSON + $json_text = Encode::encode('UTF-8', $json_text); + $stock_json_string = decode_json($json_text); + }; + if ($@) { + warn "Failed to decode JSON at index $i: $@"; next; # Skip this iteration in case of error } + my $desired_uniquename = $stock_json_string->{'uniquename'}; - if($stock_uniquename eq $desired_uniquename){ + if ($stock_uniquename eq $desired_uniquename) { push @matches, $all_audits[$i]; } } @@ -144,27 +151,33 @@ sub retrieve_trial_audits : Path('/ajax/audit/retrieve_trial_audits'){ my @matches; my $num_matches = 0; #this is to make sure only matched audits go into the matches array - for (my $i = 0; $i<$counter; $i++){ + for (my $i = 0; $i < $counter; $i++) { my $operation = $all_audits[$i][1]; my $json_string; - if($operation eq "DELETE"){ - $json_string = decode_json(Encode::decode('UTF-8', $before[$i])); - }else{ - $json_string = decode_json(Encode::decode('UTF-8', $after[$i])); + + eval { + my $json_text = ($operation eq "DELETE") ? $before[$i] : $after[$i]; + + # Convert Perl Unicode string to UTF-8 encoded bytes before decoding JSON + $json_text = Encode::encode('UTF-8', $json_text); + $json_string = decode_json($json_text); + }; + if ($@) { + warn "Failed to decode JSON at index $i: $@"; + next; # Skip this iteration in case of error } + my $desired_trial_id = $json_string->{'project_id'}; - if($trial_id eq $desired_trial_id){ - - + if ($trial_id eq $desired_trial_id) { $matches[$num_matches] = $all_audits[$i]; $num_matches++; - } } - my $match_trial_json; - $match_trial_json = encode_json(\@matches); + # Encode result properly before sending to response + my $match_trial_json = encode_json(\@matches); + utf8::encode($match_trial_json); # Convert to UTF-8 bytes $c->stash->{rest} = { match_project => $match_trial_json,