legacy servers with sharded clusters

Author: p

.travis.yml

@@ -7,6 +7,7 @@ env:
   - test=unit
   - test=api
   - test=cmd
+  - test=legacy
 
 script:
   case $test in
@@ -19,4 +20,7 @@ script:
   cmd)
     rake test:cmd
     ;;
+  legacy)
+    rake test:legacy
+    ;;
   esac

Dockerfile

@@ -24,5 +24,3 @@ COPY *.gemspec .
 RUN bundle install
 
 COPY . .
-
-CMD bundle exec rspec -f Rfc::Aif

Dockerfile.legacy

@@ -0,0 +1,31 @@
+FROM ubuntu:precise
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+# for apt-add-repository
+RUN apt-get install -y python-software-properties
+
+# https://stackoverflow.com/questions/18490591/how-to-install-ruby-2-on-ubuntu-without-rvm
+RUN apt-add-repository ppa:brightbox/ruby-ng
+
+RUN apt-get update
+RUN apt-get install -y ruby2.4 ruby2.4-dev make gcc curl libsnmp15 procps
+
+RUN mkdir /opt/mongodb
+
+RUN curl -fLo mongodb-linux-x86_64-enterprise-ubuntu1204-2.6.12.tgz \
+  https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu1204-2.6.12.tgz
+RUN tar xf mongodb-linux-x86_64-enterprise-ubuntu1204-2.6.12.tgz
+RUN mv mongo*/ /opt/mongodb/2.6
+
+ENV PATH=/opt/mongodb/2.6/bin:$PATH
+
+RUN gem install bundler --no-document
+
+COPY Gemfile .
+COPY *.gemspec .
+
+RUN bundle install
+
+COPY . .

Rakefile

@@ -15,20 +15,37 @@ task :build do
   run(%w(docker build -t mongo-manager .))
 end
 
+namespace :build do
+  task :legacy do
+    run(%w(docker build -f Dockerfile.legacy -t mongo-manager-legacy .))
+  end
+end
+
 TEST_COMMAND = %w(docker run --tmpfs /db:exec --init -it mongo-manager).freeze
+LEGACY_TEST_COMMAND = %w(docker run --tmpfs /db:exec --init -it mongo-manager-legacy).freeze
 
-task test: %w(test:unit test:api test:cmd)
+task test: %w(test:unit test:api test:cmd test:legacy)
 
 namespace :test do
   task unit: :build do
     run(TEST_COMMAND + %w(rspec -f Rfc::Aif spec/mongo_manager))
   end
 
   task api: :build do
-    run(TEST_COMMAND + %w(rspec -f Rfc::Aif spec/integration/api))
+    run(TEST_COMMAND + %w(rspec -f Rfc::Aif
+      spec/integration/api/init_spec.rb
+      spec/integration/api/init_modern_spec.rb
+    ))
   end
 
   task cmd: :build do
     run(TEST_COMMAND + %w(rspec -f Rfc::Aif spec/integration/cmd))
   end
+
+  task legacy: 'build:legacy' do
+    run(LEGACY_TEST_COMMAND + %w(rspec -f Rfc::Aif
+      spec/integration/api/init_spec.rb
+      spec/integration/api/init_legacy_spec.rb
+    ))
+  end
 end

doc/internals.md

@@ -82,3 +82,9 @@ sort of a timeout for the other nodes. If any of the other nodes are still
 starting and are not yet available, `replSetInitiate` immediately fails.
 MM works around this by waiting for each node separately before trying to
 initiate the replica set.
+
+## Sharded Clusters
+
+MongoDB 2.6 requires that the config server is a standalone and not a
+replica set. mlaunch creates a standalone config server for MongoDB 3.2 and
+lower, and replica sets for MongoDB 3.4 and higher.

lib/mongo_manager/executor.rb

@@ -129,15 +129,8 @@ def read_config
     end
 
     def init_standalone
-      cmd = [
-        mongo_path('mongod'),
-        root_dir.join('mongod.log').to_s,
-        root_dir.join('mongod.pid').to_s,
-        '--dbpath', root_dir.to_s,
-        '--port', base_port.to_s,
-      ] + server_tls_args + (options[:mongod_passthrough_args] || []) + passthrough_args
-      Helper.spawn_mongo(*cmd)
-      record_start_command(root_dir, cmd)
+      dir = root_dir.join('standalone')
+      spawn_standalone(dir, base_port, [])
 
       if options[:username]
         client = Mongo::Client.new(["localhost:#{base_port}"],
@@ -148,12 +141,9 @@ def init_standalone
 
         do_stop
 
-        cmd << '--auth'
-
-        Helper.spawn_mongo(*cmd)
+        spawn_standalone(dir, base_port, %w(--auth))
       end
 
-      record_start_command(root_dir, cmd)
       write_config
     end
 
@@ -202,14 +192,18 @@ def init_rs
     def init_sharded
       maybe_create_key
 
-      spawn_replica_set_node(
-        root_dir.join('csrs'),
-        base_port + num_mongos,
-        'csrs',
-        common_args + %w(--configsvr),
-      )
+      if config[:csrs]
+        spawn_replica_set_node(
+          root_dir.join('csrs'),
+          base_port + num_mongos,
+          'csrs',
+          common_args + %w(--configsvr),
+        )
 
-      initiate_replica_set(%W(localhost:#{base_port+num_mongos}), 'csrs', configsvr: true)
+        initiate_replica_set(%W(localhost:#{base_port+num_mongos}), 'csrs', configsvr: true)
+      else
+        spawn_standalone(root_dir.join('csrs'), base_port + num_mongos, %w(--configsvr))
+      end
 
       shard_base_port = base_port + num_mongos + 1
 
@@ -226,6 +220,12 @@ def init_sharded
         initiate_replica_set(%W(localhost:#{port}), shard_name)
       end
 
+      config_db_opt = if options[:csrs]
+        "csrs/localhost:#{base_port+num_mongos}"
+      else
+        "localhost:#{base_port+num_mongos}"
+      end
+
       1.upto(num_mongos) do |mongos|
         port = base_port - 1 + mongos
         dir = root_dir.join('router%02d' % mongos)
@@ -236,7 +236,7 @@ def init_sharded
           dir.join('mongos.log').to_s,
           dir.join('mongos.pid').to_s,
           '--port', port.to_s,
-          '--configdb', "csrs/localhost:#{base_port+num_mongos}",
+          '--configdb', config_db_opt,
         ] + server_tls_args + common_args +
           passthrough_args + (options[:mongos_passthrough_args] || [])
         Helper.spawn_mongo(*cmd)
@@ -422,6 +422,21 @@ def client_tls_options
       end.freeze
     end
 
+    def spawn_standalone(dir, port, args)
+      puts("Spawn mongod on port #{port}")
+      FileUtils.mkdir_p(dir)
+      cmd = [
+        mongo_path('mongod'),
+        dir.join('mongod.log').to_s,
+        dir.join('mongod.pid').to_s,
+        '--dbpath', dir.to_s,
+        '--port', port.to_s,
+      ] + args + server_tls_args +
+        (options[:mongod_passthrough_args] || []) + passthrough_args
+      Helper.spawn_mongo(*cmd)
+      record_start_command(dir, cmd)
+    end
+
     def spawn_replica_set_node(dir, port, replica_set_name, args)
       puts("Spawn mongod on port #{port}")
       FileUtils.mkdir(dir)

lib/mongo_manager/main.rb

@@ -57,6 +57,13 @@ def init(argv)
           options[:mongos] = v.to_i
         end
 
+        opts.on('--csrs NUM', Integer, 'Use a config server replica set with NUM nodes') do |v|
+          unless v.to_i > 0
+            usage("invalid --csrs value: #{v}")
+          end
+          options[:csrs] = v.to_i
+        end
+
         opts.on('--bin-dir DIR', String, 'Path to mongod/mongos binaries') do |v|
           options[:bin_dir] = v
         end