v3.5

Author: komstat

JGA_metadata.xlsx

@@ -3,8 +3,8 @@
 ## 日本語
 
 * 生命情報・DDBJ センター
-* 公開日: 2025-05-19
-* version: v3.4
+* 公開日: 2025-06-23
+* version: v3.5
 
 [Bioinformation and DDBJ Center](https://www.ddbj.nig.ac.jp/index-e.html) のデータベースに登録するためのメタデータ XML を生成、チェックするツール。
 * [DDBJ Sequence Read Archive (DRA)](https://www.ddbj.nig.ac.jp/dra/submission.html): Submission、Experiment、Run と Analysis (任意) XML を生成・チェックするためのエクセルとスクリプト
@@ -13,6 +13,7 @@
 
 ## 履歴
 
+* 2025-06-23: v3.5 Non-ASCII, DRA Experiment Library Layout
 * 2025-05-19: v3.4 5000 以上のオブジェクト数でワーニング
 * 2025-02-27: v3.3 Organization bug fix
 * 2025-01-16: v3.2 TEL 削除
@@ -331,8 +332,8 @@ TBD
 ## English
 
 * Bioinformation and DDBJ Center
-* release: 2025-05-19
-* version: v3.4
+* release: 2025-06-23
+* version: v3.5
 
 These files are Excel, container images and tools for generation and validation of metadata XML files for databases of [Bioinformation and DDBJ Center](https://www.ddbj.nig.ac.jp/index-e.html).
 * [DDBJ Sequence Read Archive (DRA)](https://www.ddbj.nig.ac.jp/dra/submission-e.html): generate and check Submission, Experiment and Run XML files.
@@ -341,6 +342,7 @@ These files are Excel, container images and tools for generation and validation
 
 ## History
 
+* 2025-06-23: v3.5 Non-ASCII, DRA Experiment Library Layout
 * 2025-05-19: v3.4 Warning to objects > 5000
 * 2025-02-27: v3.3 Organization bug fix
 * 2025-01-16: v3.2 TEL removed

README.md

@@ -20,6 +20,7 @@ require 'date'
 # 2023-12-21 version 2.2 center name changes
 # 2024-07-05 version 3.0 new sheet names with DB prefixes
 # 2025-05-19 version 3.1 Warnings to >5,000 objects
+# 2025-06-23 version 3.2 Non-ASCII, error when layout is missing
 #
 
 # Options
@@ -85,6 +86,30 @@ def clean_number(num)
 
 end
 
+## hash や array の全ての値が ascii かどうか
+def collect_nonascii(value, path = [])
+  case value
+  when String
+    if value.ascii_only?
+      nil
+    else
+      # 非ASCII文字だけを抽出
+      nonascii_chars = value.scan(/[^\x00-\x7F]/).uniq
+      { path: path.join("."), value: value, nonascii: nonascii_chars }
+    end
+  when Array
+    value.each_with_index.map do |v, i|
+      collect_nonascii(v, path + ["[#{i}]"])
+    end.compact
+  when Hash
+    value.map do |k, v|
+      collect_nonascii(v, path + [k.to_s])
+    end.compact
+  else
+    nil
+  end
+end
+
 ## Settings
 # XML instruction
 instruction = '<?xml version="1.0" encoding="UTF-8"?>'
@@ -279,6 +304,50 @@ puts "\nWARNING: More than 5,000 Experiments. Split your submission to keep the
 puts "\nWARNING: More than 5,000 Runs. Split your submission to keep the number of Runs <= 5,000" if runs_a.size > 5000
 puts "\nWARNING: More than 5,000 Analyses. Split your submission to keep the number of Analyses <= 5,000" if analyses_a.size > 5000
 
+## Non-ASCII
+## Non-ASCII check
+submission_nonascii = collect_nonascii(submission_a).flatten.compact
+if !submission_a.empty? && !submission_nonascii.empty?
+  submission_nonascii.each do |r|
+    puts "Non-ASCII found at Submission: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+experiment_nonascii = collect_nonascii(experiments_a).flatten.compact
+if !experiments_a.empty? && !submission_nonascii.empty?
+  experiment_nonascii.each do |r|
+    puts "Non-ASCII found at Experiment: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+run_nonascii = collect_nonascii(runs_a).flatten.compact
+if !runs_a.empty? && !run_nonascii.empty?
+  run_nonascii.each do |r|
+    puts "Non-ASCII found at Experiment: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+analysis_nonascii = collect_nonascii(analyses_a).flatten.compact
+if !analyses_a.empty? && !analysis_nonascii.empty?
+  analysis_nonascii.each do |r|
+    puts "Non-ASCII found at Analysis: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+runfile_nonascii = collect_nonascii(run_files_a).flatten.compact
+if !run_files_a.empty? && !runfile_nonascii.empty?
+  runfile_nonascii.each do |r|
+    puts "Non-ASCII found at Run file: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+analysis_nonascii = collect_nonascii(analyses_a).flatten.compact
+if !analyses_a.empty? && !analysis_nonascii.empty?
+  analysis_nonascii.each do |r|
+    puts "Non-ASCII found at Analysis: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
 ## Create XML
 prefix = submission_id + "_"
 
@@ -428,9 +497,11 @@ unless analysis_only
               elsif exp[9] =~ /paired/
                 layout.PAIRED
                 paired_seq = "paired end sequencing"
-              else
+              elsif exp[9] =~ /single/
                 layout.SINGLE
                 paired_seq = "sequencing"
+              else
+                raise "Invalid Library Layout: #{exp[0]}"
               end
             } # layout
 

exe/excel2xml_dra

@@ -11,6 +11,7 @@ require 'optparse'
 #
 
 # Update history
+# 2025-06-23 non-ascii check
 # 2025-01-16 phone removed
 # 2024-07-05 DB prefix added to sheet name
 # 2024-07-04 Record all sample attributes in the sample attribute elements
@@ -69,6 +70,30 @@ def clean_number(num)
 
 end
 
+## hash や array の全ての値が ascii かどうか
+def collect_nonascii(value, path = [])
+  case value
+  when String
+    if value.ascii_only?
+      nil
+    else
+      # 非ASCII文字だけを抽出
+      nonascii_chars = value.scan(/[^\x00-\x7F]/).uniq
+      { path: path.join("."), value: value, nonascii: nonascii_chars }
+    end
+  when Array
+    value.each_with_index.map do |v, i|
+      collect_nonascii(v, path + ["[#{i}]"])
+    end.compact
+  when Hash
+    value.map do |k, v|
+      collect_nonascii(v, path + [k.to_s])
+    end.compact
+  else
+    nil
+  end
+end
+
 ### Settings
 # instruction
 instruction = '<?xml version="1.0" encoding="UTF-8"?>'
@@ -633,6 +658,56 @@ puts "Dataset aliases duplication: #{dataset_aliases_duplicated_a.join(",")}" if
 # objects into an array
 metadata_a = [submission_a, study_a, sample_a, experiment_a, data_a, analysis_a, dataset_a]
 
+## Non-ASCII check
+submission_nonascii = collect_nonascii(submission_h).flatten.compact
+if !submission_h.empty? && !submission_nonascii.empty?
+  submission_nonascii.each do |r|
+    puts "Non-ASCII found at Submission: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+study_nonascii = collect_nonascii(study_h).flatten.compact
+if !study_h.empty? && !study_nonascii.empty?
+  study_nonascii.each do |r|
+    puts "Non-ASCII found at Study: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+dataset_nonascii = collect_nonascii(datasets_a).flatten.compact
+if !datasets_a.empty? && !dataset_nonascii.empty?
+  dataset_nonascii.each do |r|
+    puts "Non-ASCII found at Dataset: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+sample_nonascii = collect_nonascii(samples_a).flatten.compact
+if !samples_a.empty? && !sample_nonascii.empty?
+  sample_nonascii.each do |r|
+    puts "Non-ASCII found at Sample: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+experiment_nonascii = collect_nonascii(experiments_a).flatten.compact
+if !experiments_a.empty? && !experiment_nonascii.empty?
+  experiment_nonascii.each do |r|
+    puts "Non-ASCII found at Experiment: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+data_nonascii = collect_nonascii(datas_a).flatten.compact
+if !datas_a.empty? && !data_nonascii.empty?
+  data_nonascii.each do |r|
+    puts "Non-ASCII found at Data: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
+analysis_nonascii = collect_nonascii(analyses_a).flatten.compact
+if !analyses_a.empty? && !analysis_nonascii.empty?
+  analysis_nonascii.each do |r|
+    puts "Non-ASCII found at Analysis: '#{r[:value]}' (chars: #{r[:nonascii].join})"
+  end
+end
+
 ### Create XML
 prefix = submission_id + "_"
 
@@ -909,7 +984,7 @@ study_f.puts xml_study.STUDY_SET{|study_set|
 # Sample
 sample_f.puts xml_sample.SAMPLE_SET{|sample_set|
 
-  for sam in samples_a    		
+  for sam in samples_a
 
 		sample_set.SAMPLE("accession" => "", "center_name" => center_name, "alias" => sam[0]){|sample|
       sample.TITLE(sam[2].to_s.strip)