diff --git a/.rubocop.yml b/.rubocop.yml index 8361b44..7a7cdf6 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,5 +1,7 @@ AllCops: - TargetRubyVersion: 2.3 + TargetRubyVersion: 3.3 + NewCops: enable + Style/WordArray: EnforcedStyle: brackets @@ -10,6 +12,9 @@ Style/MutableConstant: Style/MixinUsage: Enabled: False +Layout/LineLength: + Max: 159 + # not my style ;-) Layout/SpaceAroundEqualsInParameterDefault: Enabled: False @@ -22,16 +27,15 @@ Style/StringLiterals: Exclude: - 'Gemfile' +Style/OptionalBooleanParameter: + Enabled: False + # we want that for better Style/TrailingCommaInHashLiteral: Enabled: False - Metrics/CyclomaticComplexity: - Max: 20 - -Metrics/LineLength: - Max: 159 + Max: 22 Metrics/ClassLength: Max: 300 @@ -43,13 +47,13 @@ Metrics/BlockLength: Max: 65 Metrics/AbcSize: - Max: 60 + Max: 70 Metrics/BlockNesting: Max: 5 Metrics/PerceivedComplexity: - Max: 20 + Max: 22 Style/ClassVars: Exclude: diff --git a/Gemfile b/Gemfile index f164db6..f145e01 100644 --- a/Gemfile +++ b/Gemfile @@ -16,14 +16,14 @@ gem "ruby-xz", "~> 1.0", group: %i[build test] gem "bzip2-ffi", "~> 1.0", group: %i[build test] # Added at 2018-09-07 16:41:14 +0200 by markus: -gem "test-unit", "~> 3.2", group: [:test] +gem "test-unit", "~> 3.5", group: [:test] # Added at 2021-11-12 17:19:23 +0200 by bernhard: gem 'parallel', '~> 1.20', '< 1.21', group: %i[build test] # Added at 2018-12-05 19:28:10 +0100 by markus: group :rubocop, optional: true do - gem "rubocop", "~> 0.61.0" + gem "rubocop", "~> 1.51.0" end group :development, optional: true do diff --git a/Gemfile.lock b/Gemfile.lock index 70d2b05..052c413 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,29 +6,34 @@ GEM bzip2-ffi (1.1.1) ffi (~> 1.0) ffi (1.16.3) - jaro_winkler (1.5.6) + json (2.7.2) parallel (1.20.1) parser (3.3.1.0) ast (~> 2.4.1) racc power_assert (2.0.3) - powerpack (0.1.3) racc (1.7.3) rainbow (3.1.1) - rubocop (0.61.1) - jaro_winkler (~> 1.5.1) + regexp_parser (2.9.1) + rexml (3.2.6) + rubocop (1.51.0) + json (~> 2.3) parallel (~> 1.10) - parser (>= 2.5, != 2.5.1.1) - powerpack (~> 0.1) + parser (>= 3.2.0.0) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.4.0) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) ruby-debian (0.3.8) ruby-progressbar (1.13.0) ruby-xz (1.0.3) test-unit (3.6.2) power_assert - unicode-display_width (1.4.1) + unicode-display_width (2.5.0) PLATFORMS x86_64-linux @@ -37,10 +42,10 @@ DEPENDENCIES byebug (~> 11.1) bzip2-ffi (~> 1.0) parallel (~> 1.20, < 1.21) - rubocop (~> 0.61.0) + rubocop (~> 1.51.0) ruby-debian (~> 0.3.8) ruby-xz (~> 1.0) - test-unit (~> 3.2) + test-unit (~> 3.5) BUNDLED WITH 2.5.9 diff --git a/debRelease.rb b/debRelease.rb index 8837ba6..793f30c 100755 --- a/debRelease.rb +++ b/debRelease.rb @@ -13,10 +13,10 @@ class DebRelease @@tempdir = '/tmp/errata_parser_cache/debian' attr_reader :data, :files - attr_accessor :suite, :base_url + attr_accessor :suite, :base_url, :whitelist_arch, :whitelist_comp attr_writer :release_name - attr_accessor :whitelist_arch, :whitelist_comp - RE_FILES = /^\s*(?[0-9a-f]+)\s+(?\d+)\s*(?\S.*)$/.freeze + + RE_FILES = /^\s*(?[0-9a-f]+)\s+(?\d+)\s*(?\S.*)$/ def initialize(uri=nil, suite='stable') init @@ -58,7 +58,7 @@ def parse(release_text) when 'date', 'valid-until' Time.parse value when 'architectures', 'components' - value.split ' ' + value.split when 'md5sum', 'sha1', 'sha256' state = key next @@ -93,22 +93,20 @@ def get_package(component, architecture) paths = paths_exist + (paths - paths_exist) paths.each do |p| - begin - basefilename = p.split('/').last - path = "#{cache_dir}/#{basefilename}" - data = download_file_cached "#{release_base_url}/#{p}", path - plainfile = "#{cache_dir}/Packages.plain" - File.open(plainfile, 'w') do |f| - case basefilename.downcase - when 'packages.xz' - require 'xz' - f << XZ.decompress(data) - when 'packages.gz' - require 'zlib' - f << Zlib.gunzip(data) - else - f << data - end + basefilename = p.split('/').last + path = "#{cache_dir}/#{basefilename}" + data = download_file_cached "#{release_base_url}/#{p}", path + plainfile = "#{cache_dir}/Packages.plain" + File.open(plainfile, 'w') do |f| + case basefilename.downcase + when 'packages.xz' + require 'xz' + f << XZ.decompress(data) + when 'packages.gz' + require 'zlib' + f << Zlib.gunzip(data) + else + f << data end return Debian::Packages.new(plainfile) rescue StandardError => e @@ -117,6 +115,12 @@ def get_package(component, architecture) ensure File.unlink plainfile if plainfile && File.exist?(plainfile) end + return Debian::Packages.new(plainfile) + rescue StandardError => e + warn "#{e} for #{p.inspect}" + FileUtils.rm_f path + ensure + File.unlink plainfile if plainfile && File.exist?(plainfile) end end diff --git a/downloader.rb b/downloader.rb index d3157f0..0b40ca8 100644 --- a/downloader.rb +++ b/downloader.rb @@ -38,7 +38,8 @@ def download_file_cached(url, path = nil, force = false, maxhop = MAXREDIRECTHOP end end - if res.is_a? Net::HTTPSuccess + case res + when Net::HTTPSuccess mode = 'wb' body = res.body # check for content type; use 'wb' for images @@ -53,11 +54,11 @@ def download_file_cached(url, path = nil, force = false, maxhop = MAXREDIRECTHOP io.write body end end - return body - elsif res.is_a? Net::HTTPNotModified + body + when Net::HTTPNotModified # Use already downloaded version - return File.read path - elsif res.is_a? Net::HTTPFound + File.read path + when Net::HTTPFound # Redirect! warn "REDIRECTed to #{res['location'].inspect}" raise "Max redirect-depth reached (#{MAXREDIRECTHOPS} hops)" if maxhop.zero? diff --git a/errata_parser.rb b/errata_parser.rb index 2f96701..0d106dd 100755 --- a/errata_parser.rb +++ b/errata_parser.rb @@ -14,7 +14,7 @@ 'tempdir' => '/tmp/errata_parser_cache' }.freeze -def fatal(message, code=42, show_help=false) +def fatal(message, code=42, show_help: false) warn message warn @opts if show_help exit code @@ -103,8 +103,8 @@ def write_errata_file(filename, errata, name: '', verbose: false, remove_empty_p write_json_file( filename, data, - name: name, - verbose: verbose + name:, + verbose: ) end @@ -138,7 +138,7 @@ def load_config(filename) ## Sanity checks fatal('No Errata-type specified!', 2, true) unless options.key?(:ubuntu) || options.key?(:ubuntu_esm) || options.key?(:debian) - parser = DebianErrataParser.new(options[:verbose]) + parser = DebianErrataParser.new(verbose: options[:verbose]) extend Downloader if options.key? :debian @@ -201,8 +201,8 @@ def load_config(filename) mutex.synchronize do # save Meta-data metadata[:releases][deb_rel.release_name] = { - 'architectures': deb_rel.architectures, - 'components': deb_rel.components + architectures: deb_rel.architectures, + components: deb_rel.components } metadata[:releases][deb_rel.release_name][:aliases] = cfg['aliases']['releases'][deb_rel.release_name] if cfg.key?('aliases') && cfg['aliases'].key?('releases') && cfg['aliases']['releases'].key?(deb_rel.release_name) @@ -286,30 +286,28 @@ def load_config(filename) repository['releases'].each do |s| threads << Thread.new do - begin - Thread.current[:repo_url] = repository['repo_url'] - if repository.key?('credentials') - url = URI.parse(Thread.current[:repo_url]) - url.user = repository['credentials']['user'] - url.password = repository['credentials']['pass'] - Thread.current[:repo_url] = url.to_s - end - warn "START Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] - deb_rel = DebRelease.new(Thread.current[:repo_url], s) - deb_rel.release_name = fix_release(deb_rel.release_name, cfg['aliases']) if deb_rel.release_name.include? '-' - deb_rel.whitelist_comp = get_whitelist(cfg, 'components') - deb_rel.whitelist_arch = whitelist_arch - pkgs = deb_rel.all_packages - - # merge package-list - mutex.synchronize do - DebRelease.assemble_ubuntu_packages(packages, pkgs) - end - warn "FINISH Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] - rescue Net::HTTPServerException => e - warn "FAILED Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}: #{e}" if options[:verbose] - raise e + Thread.current[:repo_url] = repository['repo_url'] + if repository.key?('credentials') + url = URI.parse(Thread.current[:repo_url]) + url.user = repository['credentials']['user'] + url.password = repository['credentials']['pass'] + Thread.current[:repo_url] = url.to_s + end + warn "START Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] + deb_rel = DebRelease.new(Thread.current[:repo_url], s) + deb_rel.release_name = fix_release(deb_rel.release_name, cfg['aliases']) if deb_rel.release_name.include? '-' + deb_rel.whitelist_comp = get_whitelist(cfg, 'components') + deb_rel.whitelist_arch = whitelist_arch + pkgs = deb_rel.all_packages + + # merge package-list + mutex.synchronize do + DebRelease.assemble_ubuntu_packages(packages, pkgs) end + warn "FINISH Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] + rescue Net::HTTPClientException => e + warn "FAILED Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}: #{e}" if options[:verbose] + raise e end end @@ -379,31 +377,29 @@ def load_config(filename) repository['releases'].each do |s| threads << Thread.new do - begin - Thread.current[:repo_url] = repository['repo_url'] - if repository.key?('credentials') - url = URI.parse(Thread.current[:repo_url]) - url.user = repository['credentials']['user'] - url.password = repository['credentials']['pass'] - Thread.current[:repo_url] = url.to_s - end - warn "START Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] - deb_rel = DebRelease.new(Thread.current[:repo_url], s) - deb_rel.release_name = fix_release(deb_rel.release_name, cfg['aliases']) if deb_rel.release_name.include? '-' - deb_rel.whitelist_comp = get_whitelist(cfg, 'components') - deb_rel.whitelist_arch = whitelist_arch - pkgs = deb_rel.all_packages - - # merge package-list - mutex.synchronize do - # ESM-Errata need debian style package-struct - DebRelease.assemble_debian_packages(packages_by_name, pkgs) - end - warn "FINISH Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] - rescue Net::HTTPServerException => e - warn "FAILED Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}: #{e}" if options[:verbose] - raise e + Thread.current[:repo_url] = repository['repo_url'] + if repository.key?('credentials') + url = URI.parse(Thread.current[:repo_url]) + url.user = repository['credentials']['user'] + url.password = repository['credentials']['pass'] + Thread.current[:repo_url] = url.to_s + end + warn "START Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] + deb_rel = DebRelease.new(Thread.current[:repo_url], s) + deb_rel.release_name = fix_release(deb_rel.release_name, cfg['aliases']) if deb_rel.release_name.include? '-' + deb_rel.whitelist_comp = get_whitelist(cfg, 'components') + deb_rel.whitelist_arch = whitelist_arch + pkgs = deb_rel.all_packages + + # merge package-list + mutex.synchronize do + # ESM-Errata need debian style package-struct + DebRelease.assemble_debian_packages(packages_by_name, pkgs) end + warn "FINISH Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}" if options[:verbose] + rescue Net::HTTPClientException => e + warn "FAILED Download #{s.inspect} from #{Thread.current[:repo_url].sub(/:[^:@]+@/, ':*****@')}: #{e}" if options[:verbose] + raise e end end diff --git a/gen_errata.rb b/gen_errata.rb index 4f25cc8..83d2744 100755 --- a/gen_errata.rb +++ b/gen_errata.rb @@ -6,7 +6,6 @@ require 'time' require 'debian' require 'pathname' -require 'set' require_relative 'parse_dsalist' require_relative 'downloader' @@ -44,8 +43,7 @@ def initialize(message, id=nil, release=nil, package=nil) # The erratum main class class Erratum - attr_accessor :title, :name, :cves, :source_package, :fixed_version - attr_accessor :dbts_bugs + attr_accessor :title, :name, :cves, :source_package, :fixed_version, :dbts_bugs attr_writer :description def initialize @@ -82,11 +80,11 @@ def add_cve(cve) def add_package(name, version, architecture: nil, release: nil, component: nil) hsh = { - name: name, - version: version, - architecture: architecture, - release: release, - component: component + name:, + version:, + architecture:, + release:, + component: } @packages << hsh unless @packages.include? hsh end @@ -196,7 +194,7 @@ def priorized_value(old_idx, new, list, name='value') class DebianErrataParser attr_reader :info_state, :info_state_cmplt - def initialize(verbose=false) + def initialize(verbose: false) @info_state = :init @info_state_cmplt = 1 @@ -331,7 +329,7 @@ def add_packages_ubuntu(erratum, release, data, architecture_whitelist, packages versions[match['version']] || match['version'], architecture: match['arch'], component: match['comp'], - release: release + release: ) metadata_add_entry(release, match['arch'], match['comp']) elsif @verbose @@ -359,11 +357,9 @@ def gen_ubuntu_errata(usn_db, packages, packages_by_name, release_whitelist=nil, erratum.description = usn['description'] if usn.key? 'cves' usn['cves'].each do |cve| - begin - erratum.add_cve cve - rescue RuntimeError => e - raise unless e.message.start_with? 'Invalid CVE' - end + erratum.add_cve cve + rescue RuntimeError => e + raise unless e.message.start_with? 'Invalid CVE' end end erratum.issued = usn['timestamp'] @@ -431,9 +427,9 @@ def add_binary_packages_from_file(errata, package_json_path, releases=nil, archi add_binary_packages( errata, JSON.parse(File.read(package_json_path)), - releases: releases, - architecture_whitelist: architecture_whitelist, - special_kernel_pkg_collection: special_kernel_pkg_collection + releases:, + architecture_whitelist:, + special_kernel_pkg_collection: ) end @@ -537,14 +533,14 @@ def get_binary_packages_for_erratum_package(source_pkg, pkg, packages, architect type = ARGV[0] parser = DebianErrataParser.new Thread.new do - STDERR.puts + $stderr.puts line = '' loop do # clean line - STDERR.print "#{' ' * line.length}\r" + $stderr.print "#{' ' * line.length}\r" line = "#{(parser.info_state_cmplt * 100).round}% #{parser.info_state}" - STDERR.print "#{line}\r" + $stderr.print "#{line}\r" sleep 0.1 end end diff --git a/parse_dsalist.rb b/parse_dsalist.rb index 67edce6..ef1515f 100644 --- a/parse_dsalist.rb +++ b/parse_dsalist.rb @@ -2,14 +2,14 @@ require 'json' -REGEX_1ST_LINE = /^\[(?[^\]]+)\]\s*(?[A-z0-9\-]+)\s*(?\S+)\s*-*\s*(?.*)$/.freeze -REGEX_CVE_LINE = /\s+{(?[^}]*)}/.freeze -REGEX_REL_LINE = /\s+\[(?[^\]]*)\]\s*-\s*(?\S+)\s*(?\S*)/.freeze -REGEX_NOT_LINE = /\s+NOTE:/.freeze +REGEX_1ST_LINE = /^\[(?[^\]]+)\]\s*(?[A-z0-9-]+)\s*(?\S+)\s*-*\s*(?.*)$/ +REGEX_CVE_LINE = /\s+{(?[^}]*)}/ +REGEX_REL_LINE = /\s+\[(?[^\]]*)\]\s*-\s*(?\S+)\s*(?\S*)/ +REGEX_NOT_LINE = /\s+NOTE:/ # Base-Parser Exception class ParserException < RuntimeError - def initialize(lnum = -1, line = nil, msg = 'ParserException', critical = true) + def initialize(lnum = -1, line = nil, msg = 'ParserException', critical: true) super("at #{lnum}: #{msg} (#{line})") @lnum = lnum @line = line @@ -49,6 +49,7 @@ def symbolize_hash_keys(hsh) class DSAList < Array def initialize @opt_ignore_empty_cve = true + super end def secure_push(item) @@ -157,38 +158,36 @@ def self.parse_dsa_list(io) i = 0 io.each_line do |line| - begin - i += 1 - res1 = REGEX_1ST_LINE.match(line) - if res1 - dsa_list.secure_push dsa - dsa = DSA.new(**symbolize_hash_keys(res1.named_captures)) - elsif dsa - res = REGEX_REL_LINE.match(line) - if res - dsa.add_release(**symbolize_hash_keys(res.named_captures)) - next - end - res = REGEX_CVE_LINE.match(line) - if res - dsa.add_cve(res[:cves].split(' ')) unless res[:cves].empty? - next - end - if REGEX_NOT_LINE.match(line) - # ignore 'NOTE:' lines - next - end - - raise ParserWarning.new(i, line, 'Unknown Line in DSA') - - else - raise ParserWarning.new(i, line, 'Unknown Line') + i += 1 + res1 = REGEX_1ST_LINE.match(line) + if res1 + dsa_list.secure_push dsa + dsa = DSA.new(**symbolize_hash_keys(res1.named_captures)) + elsif dsa + res = REGEX_REL_LINE.match(line) + if res + dsa.add_release(**symbolize_hash_keys(res.named_captures)) + next + end + res = REGEX_CVE_LINE.match(line) + if res + dsa.add_cve(res[:cves].split) unless res[:cves].empty? + next + end + if REGEX_NOT_LINE.match(line) + # ignore 'NOTE:' lines + next end - rescue ParserException => e - raise if e.critical - warn(e) + raise ParserWarning.new(i, line, 'Unknown Line in DSA') + + else + raise ParserWarning.new(i, line, 'Unknown Line') end + rescue ParserException => e + raise if e.critical + + warn(e) end dsa_list.secure_push dsa diff --git a/test/gen_test_data.rb b/test/gen_test_data.rb index 0a41bc0..d7f6ad1 100644 --- a/test/gen_test_data.rb +++ b/test/gen_test_data.rb @@ -62,9 +62,7 @@ class Download cve_list_filtered[package] = cves_filtered unless cves_filtered.empty? end -File.open(File.join(data_path, 'cve.json'), 'w') do |f| - f.write JSON.dump cve_list_filtered -end +File.write(File.join(data_path, 'cve.json'), JSON.dump(cve_list_filtered)) # gen USN-database warn 'receive USN list (Ubuntu)...'