Land rapid7#18704, Leverage the module metadata cache in the module_sets

Author: adfoster-r7

.github/workflows/verify.yml

@@ -76,7 +76,10 @@ jobs:
         include:
           - os: ubuntu-latest
             ruby: '3.1'
-            test_cmd: 'bundle exec rake rspec-rerun:spec SPEC_OPTS="--tag content" DATASTORE_FALLBACKS=1'
+            test_cmd: 'bundle exec rake rspec-rerun:spec SPEC_OPTS="--tag content" MSF_FEATURE_DATASTORE_FALLBACKS=1'
+          - os: ubuntu-latest
+            ruby: '3.1'
+            test_cmd: 'bundle exec rake rspec-rerun:spec SPEC_OPTS="--tag content" MSF_FEATURE_DEFER_MODULE_LOADS=1'
         test_cmd:
           - bundle exec rake rspec-rerun:spec SPEC_OPTS="--tag content"
           - bundle exec rake rspec-rerun:spec SPEC_OPTS="--tag ~content"

lib/msf/core/encoded_payload.rb

@@ -140,7 +140,7 @@ def encode
       # as the framework's list of encoder names so we can compare them later.
       # This is important for when we get input from RPC.
       if reqs['Encoder']
-        reqs['Encoder'] = reqs['Encoder'].encode(framework.encoders.keys[0].encoding)
+        reqs['Encoder'] = reqs['Encoder'].encode(framework.encoders.module_refnames[0].encoding)
       end
 
       # If the caller had a preferred encoder, use this encoder only
@@ -237,9 +237,9 @@ def encode
 
           begin
             eout = self.encoder.encode(eout, reqs['BadChars'], nil, pinst.platform)
-          rescue EncodingError
-            wlog("#{err_start}: Encoder #{encoder.refname} failed: #{$!}", 'core', LEV_1)
-            dlog("#{err_start}: Call stack\n#{$@.join("\n")}", 'core', LEV_3)
+          rescue EncodingError => e
+            wlog("#{err_start}: Encoder #{encoder.refname} failed: #{e}", 'core', LEV_1)
+            dlog("#{err_start}: Call stack\n#{e.backtrace}", 'core', LEV_3)
             next_encoder = true
             break
 
@@ -342,7 +342,7 @@ def generate_sled
         wlog("#{pinst.refname}: Failed to find preferred nop #{reqs['Nop']}")
       end
 
-      nops.each { |nopname, nopmod|
+      nops.each_module { |nopname, nopmod|
         # Create an instance of the nop module
         self.nop = nopmod.new
 

lib/msf/core/exploit/remote/browser_autopwn2.rb

@@ -48,9 +48,8 @@ module Exploit::Remote::BrowserAutopwn2
     # @return [void]
     def init_exploits
       # First we're going to avoid using #find_all because that gets very slow.
-      framework.exploits.each_pair do |fullname, place_holder|
-        # If the place holder isn't __SYMBOLIC__, then that means the module is initialized,
-        # and that's gotta be the active browser autopwn.
+      framework.exploits.module_refnames.each do |fullname|
+
         next if !fullname.include?('browser') || self.fullname == "exploit/#{fullname}"
 
         # The user gets to specify which modules to include/exclude
@@ -269,7 +268,7 @@ def get_selected_payload_name(platform)
 
       # The payload is legit, we can use it.
       # Avoid #create seems faster
-      return payload_name if framework.payloads.keys.include?(payload_name)
+      return payload_name if framework.payloads.module_refnames.include?(payload_name)
 
       default = DEFAULT_PAYLOADS[platform][:payload]
 

lib/msf/core/module/platform.rb

@@ -131,7 +131,6 @@ def self.build_child_platform_abbrev(mod)
   # the string).
   #
   def self.find_portion(mod, str)
-
     # Check to see if we've built the abbreviated cache
     if (not (
           mod.const_defined?('Abbrev') and

lib/msf/core/module/platform_list.rb

@@ -25,11 +25,7 @@ def self.win32
   # convenient.
   #
   def self.transform(src)
-    if (src.kind_of?(Array))
-      from_a(src)
-    else
-      from_a([src])
-    end
+    from_a(Array.wrap(src))
   end
 
   #

lib/msf/core/module_set.rb

@@ -20,11 +20,11 @@ class Msf::ModuleSet < Hash
   # and then returns the now-loaded class afterwards.
   #
   # @param [String] name the module reference name
-  # @return [Msf::Module] instance of the of the Msf::Module subclass with the given reference name
+  # @return [Msf::Module] Class of the of the Msf::Module with the given reference name
   def [](name)
-    module_instance = super
-    if module_instance == Msf::SymbolicModule || module_instance.nil?
-      create(name)
+    module_class = super
+    if module_class == Msf::SymbolicModule || module_class.nil?
+      load_module_class(name)
     end
 
     super
@@ -36,14 +36,8 @@ def [](name)
   # @return [Msf::Module,nil] Instance of the named module or nil if it
   #   could not be created.
   def create(reference_name, cache_type: Msf::ModuleManager::Cache::FILESYSTEM)
-    klass = fetch(reference_name, nil)
+    klass = load_module_class(reference_name, cache_type: cache_type)
     instance = nil
-    # If there is no module associated with this class, then try to demand load it.
-    if klass.nil? or klass == Msf::SymbolicModule
-      framework.modules.load_cached_module(module_type, reference_name, cache_type: cache_type)
-      klass = fetch(reference_name, nil)
-    end
-
     # If the klass is valid for this reference_name, try to create it
     unless klass.nil? or klass == Msf::SymbolicModule
       instance = klass.new
@@ -56,7 +50,7 @@ def create(reference_name, cache_type: Msf::ModuleManager::Cache::FILESYSTEM)
       self.delete(reference_name)
     end
 
-    return instance
+    instance
   end
 
   # Overrides the builtin 'each' operator to avoid the following exception on Ruby 1.9.2+
@@ -68,7 +62,7 @@ def create(reference_name, cache_type: Msf::ModuleManager::Cache::FILESYSTEM)
   # @return [void]
   def each(&block)
     list = []
-    self.keys.sort.each do |sidx|
+    module_metadata.keys.sort.each do |sidx|
       list << [sidx, self[sidx]]
     end
     list.each(&block)
@@ -81,9 +75,7 @@ def each(&block)
   # @yieldparam (see #each_module_list)
   # @return (see #each_module_list)
   def each_module(opts = {}, &block)
-    demand_load_modules
-
-    self.mod_sorted = self.sort
+    self.mod_sorted = module_metadata.sort
 
     each_module_list(mod_sorted, opts, &block)
   end
@@ -107,8 +99,6 @@ def each_module_filter(opts, name, entry)
   # @yieldparam (see #each_module_list)
   # @return (see #each_module_list)
   def each_module_ranked(opts = {}, &block)
-    demand_load_modules
-
     each_module_list(rank_modules, opts, &block)
   end
 
@@ -166,7 +156,6 @@ def recalculate
   # @return [true] if the module can be {#create created} and cached.
   # @return [false] otherwise
   def valid?(reference_name)
-    create(reference_name)
     (self[reference_name]) ? true : false
   end
 
@@ -203,28 +192,12 @@ def add_module(klass, reference_name, info = {})
     klass
   end
 
-  protected
-
-  # Load all modules that are marked as being symbolic.
-  #
-  # @return [void]
-  def demand_load_modules
-    found_symbolics = false
-    # Pre-scan the module list for any symbolic modules
-    self.each_pair { |name, mod|
-      if (mod == Msf::SymbolicModule)
-        found_symbolics = true
-        mod = create(name)
-        next if (mod.nil?)
-      end
-    }
-
-    # If we found any symbolic modules, then recalculate.
-    if (found_symbolics)
-      recalculate
-    end
+  def module_refnames
+    module_metadata.keys
   end
 
+  protected
+
   # Enumerates the modules in the supplied array with possible limiting factors.
   #
   # @param [Array<Array<String, Class>>] ary Array of module reference name and module class pairs
@@ -238,35 +211,32 @@ def demand_load_modules
   # @yieldparam [Class] module The module class: a subclass of {Msf::Module}.
   # @return [void]
   def each_module_list(ary, opts, &block)
-    ary.each { |entry|
-      name, mod = entry
-
-      # Skip any lingering symbolic modules.
-      next if (mod == Msf::SymbolicModule)
+    ary.each do |entry|
+      name, module_metadata = entry
 
       # Filter out incompatible architectures
       if (opts['Arch'])
-        if (!architectures_by_module[mod])
-          architectures_by_module[mod] = mod.new.arch
+        if (!architectures_by_module[name])
+          architectures_by_module[name] = Array.wrap(module_metadata.arch)
         end
 
-        next if ((architectures_by_module[mod] & opts['Arch']).empty? == true)
+        next if ((architectures_by_module[name] & opts['Arch']).empty? == true)
       end
 
       # Filter out incompatible platforms
       if (opts['Platform'])
-        if (!platforms_by_module[mod])
-          platforms_by_module[mod] = mod.new.platform
+        if (!platforms_by_module[name])
+          platforms_by_module[name] = module_metadata.platform_list
         end
 
-        next if ((platforms_by_module[mod] & opts['Platform']).empty? == true)
+        next if ((platforms_by_module[name] & opts['Platform']).empty? == true)
       end
 
       # Custom filtering
       next if (each_module_filter(opts, name, entry) == true)
 
-      block.call(name, mod)
-    }
+      block.call(name, self[name])
+    end
   end
 
   # @!attribute [rw] ambiguous_module_reference_name_set
@@ -304,33 +274,23 @@ def each_module_list(ary, opts, &block)
   # @return [Array<Array<String, Class>>] Array of arrays where the inner array is a pair of the module reference name
   #   and the module class.
   def rank_modules
-    self.sort_by { |pair| module_rank(*pair) }.reverse!
+    module_metadata.sort_by do |refname, metadata|
+      [metadata.rank || Msf::NormalRanking, refname]
+    end.reverse!
   end
 
-  # Retrieves the rank from a loaded, not-yet-loaded, or unloadable Metasploit Module.
-  #
-  # @param reference_name [String] The reference name of the Metasploit Module
-  # @param metasploit_module_class [Class<Msf::Module>, Msf::SymbolicModule] The loaded `Class` for the Metasploit
-  #   Module, or {Msf::SymbolicModule} if the Metasploit Module is not loaded yet.
-  # @return [Integer] an `Msf::*Ranking`.  `Msf::ManualRanking` if `metasploit_module_class` is `nil` or
-  #   {Msf::SymbolicModule} and it could not be loaded by {#create}.  Otherwise, the `Rank` constant of the
-  #   `metasploit_module_class` or {Msf::NormalRanking} if `metasploit_module_class` does not define `Rank`.
-  def module_rank(reference_name, metasploit_module_class)
-    if metasploit_module_class.nil?
-      Msf::ManualRanking
-    elsif metasploit_module_class == Msf::SymbolicModule
-      # TODO don't create an instance just to get the Class.
-      created_metasploit_module_instance = create(reference_name)
-
-      if created_metasploit_module_instance.nil?
-        module_rank(reference_name, nil)
-      else
-        module_rank(reference_name, created_metasploit_module_instance.class)
-      end
-    elsif metasploit_module_class.const_defined? :Rank
-      metasploit_module_class.const_get :Rank
-    else
-      Msf::NormalRanking
+  def module_metadata
+    Msf::Modules::Metadata::Cache.instance.module_metadata(module_type)
+  end
+
+  def load_module_class(reference_name, cache_type: Msf::ModuleManager::Cache::FILESYSTEM)
+    klass = fetch(reference_name, nil)
+
+    # If there is no module associated with this class, then try to demand load it.
+    if klass.nil? || klass == Msf::SymbolicModule
+      framework.modules.load_cached_module(module_type, reference_name, cache_type: cache_type)
+      klass = fetch(reference_name, nil)
     end
+    klass
   end
 end

lib/msf/core/modules/metadata/cache.rb

@@ -86,6 +86,14 @@ def refresh_metadata(module_sets)
     end
   end
 
+  def module_metadata(type)
+    @mutex.synchronize do
+      wait_for_load
+      # TODO: Should probably figure out a way to cache this
+      @module_metadata_cache.filter_map { |_, metadata| [metadata.ref_name, metadata] if metadata.type == type }.to_h
+    end
+  end
+
   #######
   private
   #######
@@ -154,7 +162,7 @@ def initialize
     @module_metadata_cache = {}
     @store_loaded = false
     @console = Rex::Ui::Text::Output::Stdio.new
-    @load_thread = Thread.new  {
+    @load_thread = Thread.new {
       init_store
       @store_loaded = true
     }

lib/msf/core/modules/metadata/obj.rb

@@ -28,8 +28,10 @@ class Obj
   attr_reader :description
   # @return [Array<String>]
   attr_reader :references
-  # @return [Boolean]
+  # @return [String]
   attr_reader :platform
+  # @return [Msf::Module::PlatformList]
+  attr_reader :platform_list
   # @return [String]
   attr_reader :arch
   # @return [Integer]
@@ -90,6 +92,7 @@ def initialize(module_instance, obj_hash = nil)
     @default_credential = module_instance.default_cred?
 
     @platform           = module_instance.platform_to_s
+    @platform_list      = module_instance.platform
     # Done to ensure that differences do not show up for the same array grouping
     sort_platform_string
 
@@ -235,6 +238,7 @@ def init_from_hash(obj_hash)
     @author              = obj_hash['author'].nil? ? [] : obj_hash['author']
     @references          = obj_hash['references']
     @platform            = obj_hash['platform']
+    @platform_list       = parse_platform_list(@platform)
     @arch                = obj_hash['arch']
     @rport               = obj_hash['rport']
     @mod_time            = Time.parse(obj_hash['mod_time'])
@@ -288,6 +292,18 @@ def force_encoding(encoding)
     @references = @references.map {|r| r.dup.force_encoding(encoding)}
   end
 
+  def parse_platform_list(platform_string)
+    return nil if platform_string.nil?
+
+    if platform_string.casecmp('All')
+      # empty string represents all platforms in Msf::Module::PlatformList
+      platforms = ['']
+    else
+      platforms = platform_string.split(',')
+    end
+    Msf::Module::PlatformList.transform(platforms)
+  end
+
 end
 end
 end

lib/msf/core/payload_generator.rb

@@ -598,7 +598,7 @@ def format_is_valid?
     # @return [True] if the payload is a valid Metasploit Payload
     # @return [False] if the payload is not a valid Metasploit Payload
     def payload_is_valid?
-      (framework.payloads.keys + ['stdin']).include? payload
+      (framework.payloads.module_refnames + ['stdin']).include? payload
     end
 
   end