Add includes option to ruby_bundle rule for per-gem load path customization (#102)

* Add register_gem support for non-standard gemspecs

* cleanup and workaround

* debug

* Add includes option to bundle_build

* Add description about includes/excludes optional parameters for
ruby_bundle

* Trim extra whitespace characters

* Add example usage of ruby_bundle.includes to README

* Add a test for includes option of ruby_bundle rule

* tidy up

* Fix the default value of ruby_bundle rule

* Bump up Ruby version of ruby_bundle includes sample code

* Fix new test workspace

* Fix bundle build rule path injection

* Fix usage of ruby_bundle rule's includes option

* More clarification in comments

* Clarify that folders in spec.require_paths do not need to be listed in includes hash

Co-authored-by: Konstantin Gredeskoul <[email protected]>

Author: mmizutani

README.md

@@ -20,7 +20,7 @@
 
 Note: we have a short guide on [Building your first Ruby Project](https://github.com/bazelruby/rules_ruby/wiki/Build-your-ruby-project) on the Wiki. We encourage you to check it out.
 
-## Table of Contents 
+## Table of Contents
 
 - [Ruby Rules® for Bazel Build System](#ruby-rules-for-bazelhttpsbazelbuild-build-system)
   - [Build Status & Activity](#build-status-activity)
@@ -99,6 +99,12 @@ load(
 
 ruby_bundle(
     name = "bundle",
+    # Specify additional paths to be loaded from the gems at runtime, if any.
+    # Since spec.require_paths in Gem specifications are auto-included, directory paths
+    # in spec.require_paths do not need to be listed in includes hash.
+    includes = {
+        "grpc": ["etc"],
+    },
     excludes = {
         "mini_portile": ["test/**/*"],
     },
@@ -200,7 +206,7 @@ ruby_gem(
         "rubocop": "",
     },
     srcs = [
-	 	glob("{bin,exe,lib,spec}/**/*.rb")
+        glob("{bin,exe,lib,spec}/**/*.rb")
     ],
     deps = [
         "//lib:example_gem",
@@ -224,7 +230,7 @@ You will have to be sure to export the `ASDF_DATA_DIR` in your profile since it'
 
 ### Rule Dependency Diagram
 
-> NOTE: this diagram is slightly outdated.
+> NOTE: this diagram is somewhat outdated.
 
 The following diagram attempts to capture the implementation behind `ruby_library` that depends on the result of `bundle install`, and a `ruby_binary` that depends on both:
 
@@ -447,7 +453,7 @@ ruby_test(
     size,
     timeout,
     flaky,
-    local, 
+    local,
     shard_count
 )
 ```
@@ -540,7 +546,8 @@ ruby_bundle(
     gemfile,
     gemfile_lock,
     bundler_version = "2.1.4",
-    excludes = [],
+    includes = {},
+    excludes = {},
     vendor_cache = False,
     ruby_sdk = "@org_ruby_lang_ruby_toolchain",
     ruby_interpreter = "@org_ruby_lang_ruby_toolchain//:ruby",
@@ -598,6 +605,29 @@ ruby_bundle(
           <p>NOTE: This rule never updates the <code>Gemfile.lock</code>. It is your responsibility to generate/update <code>Gemfile.lock</code></p>
       </td>
     </tr>
+    <tr>
+      <td><code>includes</code></td>
+      <td>
+        <code>Dictionary of key-value-pairs (key: string, value: list of strings), optional</code>
+        <p>
+          List of glob patterns per gem to be additionally loaded from the library.
+          Keys are the names of the gems which require some file/directory paths not listed in the <code>require_paths</code> attribute of the gemspecs to be also added to <code>$LOAD_PATH</code> at runtime.
+          Values are lists of blob path patterns, which are relative to the root directories of the gems.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><code>excludes</code></td>
+      <td>
+        <code>Dictionary of key-value-pairs (key: string, value: list of strings), optional</code>
+        <p>
+          List of glob patterns per gem to be excluded from the library.
+          Keys are the names of the gems.
+          Values are lists of blob path patterns, which are relative to the root directories of the gems.
+          The default value is <code>["**/* *.*", "**/* */*"]</code>
+        </p>
+      </td>
+    </tr>
   </tbody>
 </table>
 
@@ -857,17 +887,17 @@ ruby_gem(
     <tr>
       <td><code>gem_description</code></td>
       <td>
-        <code>String, required</code>      
+        <code>String, required</code>
         <p>Single-line, paragraph-sized description text for the gem.</p>
       </td>
     </tr>
     <tr>
       <td><code>gem_homepage</code></td>
       <td>
-        <code>String, optional</code>      
+        <code>String, optional</code>
         <p>Homepage URL of the gem.</p>
       </td>
-    </tr>    
+    </tr>
     <tr>
       <td><code>gem_authors</code></td>
       <td>
@@ -886,7 +916,7 @@ ruby_gem(
           List of email addresses of the authors.
         </p>
       </td>
-    </tr>    
+    </tr>
     <tr>
       <td><code>srcs</code></td>
       <td>
@@ -917,7 +947,7 @@ ruby_gem(
           Typically this value is just `lib` (which is also the default).
         </p>
       </td>
-    </tr>    
+    </tr>
     <tr>
       <td><code>gem_runtime_dependencies</code></td>
       <td>
@@ -938,8 +968,8 @@ ruby_gem(
         testing gems, linters, code coverage and more.
         </p>
       </td>
-    </tr>    
-    
+    </tr>
+
   </tbody>
 </table>
 
@@ -972,13 +1002,14 @@ After that, cd into the top level folder and run the setup script in your Termin
 This runs a complete setup, shouldn't take too long. You can explore various script options with the `help` command:
 
 ```bash
-❯ bin/setup help
+❯ bin/setup -h
+
 USAGE
   # without any arguments runs a complete setup.
   bin/setup
 
   # alternatively, a sub-setup function name can be passed:
-  bin/setup [ gems | git-hook | help | os-specific | main | remove-git-hook ]
+  bin/setup [ gems | git-hook | help | main | os-specific | rbenv | remove-git-hook ]
 
 DESCRIPTION:
   Runs full setup without any arguments.
@@ -988,7 +1019,13 @@ DESCRIPTION:
   This action removes the git commit hook installed by the setup.
 
 EXAMPLES:
-    bin/setup — runs the entire setup.
+    bin/setup
+
+  Or, to run only one of the sub-functions (actions), pass
+  it as an argument:
+
+    bin/setup help
+    bin/setup remove-git-hook
 ```
 
 #### OS-Specific Setup

WORKSPACE

@@ -19,6 +19,11 @@ local_repository(
     path = "ruby/tests/testdata/another_workspace",
 )
 
+local_repository(
+    name = "bazelruby_rules_ruby_ruby_tests_testdata_bundle_includes_workspace",
+    path = "ruby/tests/testdata/bundle_includes_workspace",
+)
+
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 

ruby/private/bundle/create_bundle_build_file.rb

@@ -33,14 +33,14 @@
     srcs = glob(
       include = [
         ".bundle/config",
-        "{gem_lib_files}",
-        "lib/ruby/{ruby_version}/specifications/{name}-{version}.gemspec",
+        {gem_lib_files},
+        "{gem_spec}",
         {gem_binaries}
       ],
       exclude = {exclude},
     ),
     deps = {deps},
-    includes = ["lib/ruby/{ruby_version}/gems/{name}-{version}/lib"],
+    includes = [{gem_lib_paths}],
   )
 GEM_TEMPLATE
 
@@ -59,7 +59,11 @@
 ALL_GEMS
 
 GEM_PATH = ->(ruby_version, gem_name, gem_version) do
-  "lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}"
+  Dir.glob("lib/ruby/#{ruby_version}/gems/#{gem_name}-#{gem_version}*").first
+end
+
+SPEC_PATH = ->(ruby_version, gem_name, gem_version) do
+  Dir.glob("lib/ruby/#{ruby_version}/specifications/#{gem_name}-#{gem_version}*.gemspec").first
 end
 
 require 'bundler'
@@ -147,6 +151,7 @@ class BundleBuildFileGenerator
               :repo_name,
               :build_file,
               :gemfile_lock,
+              :includes,
               :excludes,
               :ruby_version
 
@@ -158,11 +163,14 @@ def initialize(workspace_name:,
                  repo_name:,
                  build_file: 'BUILD.bazel',
                  gemfile_lock: 'Gemfile.lock',
-                 excludes: nil)
+                 includes: nil,
+                 excludes: nil,
+                 additional_require_paths: nil)
     @workspace_name = workspace_name
     @repo_name      = repo_name
     @build_file     = build_file
     @gemfile_lock   = gemfile_lock
+    @includes       = includes
     @excludes       = excludes
     # This attribute returns 0 as the third minor version number, which happens to be
     # what Ruby uses in the PATH to gems, eg. ruby 2.6.5 would have a folder called
@@ -226,7 +234,16 @@ def remove_bundler_version!
 
   def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries)
     gem_path = GEM_PATH[ruby_version, spec.name, spec.version]
-    bundle_lib_paths << gem_lib_path = gem_path + '/lib'
+    spec_path = SPEC_PATH[ruby_version, spec.name, spec.version]
+    base_dir = "lib/ruby/#{ruby_version}"
+
+    # paths to register to $LOAD_PATH
+    require_paths = Gem::StubSpecification.gemspec_stub(spec_path, base_dir, "#{base_dir}/gems").require_paths
+    # Usually, registering the directory paths listed in the `require_paths` of gemspecs is sufficient, but
+    # some gems also require additional paths to be included in the load paths.
+    require_paths += include_array(spec.name)
+    gem_lib_paths = require_paths.map { |require_path| File.join(gem_path, require_path) }
+    bundle_lib_paths.push(*gem_lib_paths)
 
     # paths to search for executables
     gem_binaries               = find_bundle_binaries(gem_path)
@@ -237,8 +254,9 @@ def register_gem(spec, template_out, bundle_lib_paths, bundle_binaries)
     warn("registering gem #{spec.name} with binaries: #{gem_binaries}") if bundle_binaries.key?(spec.name)
 
     template_out.puts GEM_TEMPLATE
-                        .gsub('{gem_lib_path}', gem_lib_path)
-                        .gsub('{gem_lib_files}', gem_lib_path + '/**/*')
+                        .gsub('{gem_lib_paths}', to_flat_string(gem_lib_paths))
+                        .gsub('{gem_lib_files}', to_flat_string(gem_lib_paths.map { |p| "#{p}/**/*" }))
+                        .gsub('{gem_spec}', spec_path)
                         .gsub('{gem_binaries}', to_flat_string(gem_binaries))
                         .gsub('{exclude}', exclude_array(spec.name).to_s)
                         .gsub('{name}', spec.name)
@@ -265,6 +283,10 @@ def find_bundle_binaries(gem_path)
       .map { |binary| 'bin/' + binary }
   end
 
+  def include_array(gem_name)
+    (includes[gem_name] || [])
+  end
+
   def exclude_array(gem_name)
     (excludes[gem_name] || []) + DEFAULT_EXCLUDES
   end
@@ -274,18 +296,19 @@ def to_flat_string(array)
   end
 end
 
-# ruby ./create_bundle_build_file.rb "BUILD.bazel" "Gemfile.lock" "repo_name" "[]" "wsp_name"
+# ruby ./create_bundle_build_file.rb "BUILD.bazel" "Gemfile.lock" "repo_name" "{}" "{}" "wsp_name"
 if $0 == __FILE__
-  if ARGV.length != 5
-    warn("USAGE: #{$0} BUILD.bazel Gemfile.lock repo-name [excludes-json] workspace-name".orange)
+  if ARGV.length != 6
+    warn("USAGE: #{$0} BUILD.bazel Gemfile.lock repo-name {includes-json} {excludes-json} workspace-name".orange)
     exit(1)
   end
 
-  build_file, gemfile_lock, repo_name, excludes, workspace_name, * = *ARGV
+  build_file, gemfile_lock, repo_name, includes, excludes, workspace_name, * = *ARGV
 
   BundleBuildFileGenerator.new(build_file:     build_file,
                                gemfile_lock:   gemfile_lock,
                                repo_name:      repo_name,
+                               includes:       JSON.parse(includes),
                                excludes:       JSON.parse(excludes),
                                workspace_name: workspace_name).generate!
 

ruby/private/bundle/def.bzl

@@ -144,6 +144,7 @@ def generate_bundle_build_file(runtime_ctx, previous_result):
         "BUILD.bazel",  # Bazel build file (can be empty)
         "Gemfile.lock",  # Gemfile.lock where we list all direct and transitive dependencies
         runtime_ctx.ctx.name,  # Name of the target
+        repr(runtime_ctx.ctx.attr.includes),
         repr(runtime_ctx.ctx.attr.excludes),
         RULES_RUBY_WORKSPACE_NAME,
     ]

ruby/private/constants.bzl

@@ -85,6 +85,9 @@ BUNDLE_ATTRS = {
     "bundler_version": attr.string(
         default = DEFAULT_BUNDLER_VERSION,
     ),
+    "includes": attr.string_list_dict(
+        doc = "List of glob patterns per gem to be additionally loaded from the library",
+    ),
     "excludes": attr.string_list_dict(
         doc = "List of glob patterns per gem to be excluded from the library",
     ),

ruby/tests/testdata/bundle_includes_workspace/BUILD.bazel

@@ -0,0 +1,15 @@
+load(
+    "@bazelruby_rules_ruby//ruby:defs.bzl",
+    "ruby_binary",
+)
+
+package(default_visibility = ["//:__subpackages__"])
+
+ruby_binary(
+    name = "script",
+    srcs = ["script.rb"],
+    main = "script.rb",
+    deps = [
+        "@gems//:grpc",
+    ],
+)

ruby/tests/testdata/bundle_includes_workspace/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'grpc'

ruby/tests/testdata/bundle_includes_workspace/Gemfile.lock

@@ -0,0 +1,18 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    google-protobuf (3.17.3)
+    googleapis-common-protos-types (1.1.0)
+      google-protobuf (~> 3.14)
+    grpc (1.38.0)
+      google-protobuf (~> 3.15)
+      googleapis-common-protos-types (~> 1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  grpc
+
+BUNDLED WITH
+   2.2.22

ruby/tests/testdata/bundle_includes_workspace/WORKSPACE

@@ -0,0 +1,35 @@
+workspace(name = "bazelruby_rules_ruby_ruby_tests_testdata_bundle_includes_workspace")
+
+local_repository(
+    name = "bazelruby_rules_ruby",
+    path = "../../../..",
+)
+
+load(
+    "@bazelruby_rules_ruby//ruby:deps.bzl",
+    "rules_ruby_dependencies",
+    "rules_ruby_select_sdk",
+)
+
+rules_ruby_dependencies()
+
+rules_ruby_select_sdk(version = "3.0.1")
+
+load("@bazelruby_rules_ruby//ruby:defs.bzl", "ruby_bundle")
+
+ruby_bundle(
+    name = "gems",
+    bundler_version = "2.2.21",
+    gemfile = "//:Gemfile",
+    gemfile_lock = "//:Gemfile.lock",
+    includes = {
+        # The gemspec of grpc gem lists ['src/ruby/bin', 'src/ruby/lib', 'src/ruby/pb'] as the `require_paths`. When installing
+        # pre-built versions of the gem using a package downloaded from rubygems.org, these paths are sufficient since the file
+        # `src/ruby/lib/grpc.rb` in the downloaded gem package does not `require` any file outside these directories.
+        # However, when installing grpc gem from source using Bundler, `src/ruby/lib/grpc.rb` in the source package does
+        # `require` 'etc/roots.pem', so the directory containing this `require`-d file also needs to be present in the `$LOAD_PATH`.
+        # Thus users have to manually add the 'etc' directory to the `$LOAD_PATH` using the `includes` option of `ruby_bundle` rule.
+        # The `includes` option of `ruby_bundle` rule is a means of workaround for such a peculiar situation.
+        "grpc": ["etc"],
+    },
+)