Use only 2xx http requests to compute req/s (the-benchmarker#3493)

Author: waghanza

.env

@@ -0,0 +1,4 @@
+DATABASE_URL=postgresql://postgres@localhost/benchmark
+DURATION=15
+CONCURRENCIES=64,256,512
+ROUTES=GET:/,GET:/user/0,POST:/user

Gemfile

@@ -1,16 +1,17 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-group :procuction do
-  gem "mustache"
-  gem "pg"
-  gem "rake"
-  gem "dotenv"
-  gem 'activesupport'
-end
+gem 'activesupport'
+gem 'dotenv'
+gem 'mustache'
+gem 'pg'
+gem 'rake'
+
+gem 'bigdecimal'
+gem 'open3'
 
 group :development, :test do
-  gem "rspec"
-  gem "rubocop"
+  gem 'rspec'
+  gem 'rubocop'
 end

README.md

@@ -25,7 +25,7 @@ This project aims to be a load benchmarking suite, no more, no less
 
 ## Requirements
 
-+ [Crystal](https://crystal-lang.org) as `built-in` tools are made in this language
++ [Ruby](https://ruby-lang.org) as `built-in` tools are made in this language
 + [Docker](https://www.docker.com) as **frameworks** are `isolated` into _containers_
 + [wrk](https://github.com/wg/wrk) as benchmarking tool, `>= 4.1.0`
 + [postgresql](https://www.postgresql.org) to store data, `>= 10`
@@ -44,67 +44,9 @@ eval $(docker-machine env default)
 
 ## Usage
 
-+ Install all dependencies
+... to be documented ...
 
-~~~sh
-shards install
-~~~
-
-+ Build internal tools
-
-~~~sh
-shards build
-~~~
-
-+ Create and initialize the database
-
-~~~sh
-createdb -U postgres benchmark
-psql -U postgres -d benchmark < dump.sql
-~~~
-
-Docker can be used to set up the database:
-
-~~~sh
-docker run -it --rm -d \
--p 5432:5432 \
--e POSTGRES_DB=benchmark \
--e POSTGRES_HOST_AUTH_METHOD=trust \
--v /tmp/pg-data:/var/lib/postgresql/data \
---name pg postgres:12-alpine
-~~~
-
-Wait several seconds for the container to start, then inject the dump:
-
-~~~sh
-docker exec pg sh -c "echo \"$(cat dump.sql)\" | psql -U postgres -d benchmark"
-~~~
-
-After creating the database, export its URL:
-
-~~~sh
-export DATABASE_URL="postgresql://postgres@localhost/benchmark"
-~~~
-
-+ Make configuration
-
-~~~sh
-bin/make config
-~~~
-
-+ Build containers
-
-> jobs are either languages (example : crystal) or frameworks (example : router.cr)
-
-~~~sh
-bin/neph [job1] [job2] [job3] ...
-~~~
-
-+ Export all results readme
-
-~~~sh
-bin/db to_readme
-~~~
+feel free to create an issue if you want to try this project
 
 ## Results
 

README.mustache.md

@@ -4,16 +4,12 @@ require 'dotenv'
 
 Dir.glob('lib/tasks/*.rake').each { |r| load r }
 
-environment = ENV.fetch('ENV', 'development')
-
 MANIFESTS = {
   container: '.Dockerfile',
   build: '.Makefile'
 }.freeze
 
-default_environment = File.join('.env', 'default')
-custom_environment = File.join('.env', environment)
-Dotenv.load(custom_environment, default_environment)
+Dotenv.load
 
 class ::Hash
   def recursive_merge(h)
@@ -29,27 +25,20 @@ def default_provider
   end
 end
 
-def commands_for(language, framework, **options)
+def commands_for(language, framework, provider)
   config = YAML.safe_load(File.read('config.yaml'))
 
-  directory = File.dirname(options[:path])
-  main_config = YAML.safe_load(File.open(File.join(directory, '..', '..', 'config.yaml')))
-  language_config = YAML.safe_load(File.open(File.join(directory, '..', 'config.yaml')))
-  framework_config = YAML.safe_load(File.open(File.join(directory, 'config.yaml')))
+  directory = Dir.pwd
+  main_config = YAML.safe_load(File.open(File.join(directory, 'config.yaml')))
+  language_config = YAML.safe_load(File.open(File.join(directory, language, 'config.yaml')))
+  framework_config = YAML.safe_load(File.open(File.join(directory, language, framework, 'config.yaml')))
   app_config = main_config.recursive_merge(language_config).recursive_merge(framework_config)
-
-  options[:framework] = framework
-  options[:language] = language
-
-  ENV.each do |key, value|
-    options[key] = value unless options.key?(key)
-  end
-
+  options = { language: language, framework: framework }
   commands = { build: [], collect: [], clean: [] }
 
   # Compile first, only for non containers
 
-  if app_config.key?('binaries') && !(options[:provider].start_with?('docker') || options[:provider].start_with?('podman'))
+  if app_config.key?('binaries') && !(provider.start_with?('docker') || provider.start_with?('podman'))
     commands << "docker build -f #{MANIFESTS[:container]} -t #{language}.#{framework} ."
     commands << "docker run -td #{language}.#{framework} > cid.txt"
     app_config['binaries'].each do |out|
@@ -62,36 +51,32 @@ def commands_for(language, framework, **options)
     end
   end
 
-  config['providers'][options[:provider]]['build'].each do |cmd|
+  config['providers'][provider]['build'].each do |cmd|
     commands[:build] << Mustache.render(cmd, options.merge!(manifest: MANIFESTS[:container])).to_s
   end
 
-  config['providers'][options[:provider]]['metadata'].each do |cmd|
+  config['providers'][provider]['metadata'].each do |cmd|
     commands[:build] << Mustache.render(cmd, options).to_s
   end
 
-  if app_config.key?('bootstrap') && config['providers'][options[:provider]].key?('exec')
-    remote_command = config['providers'][options[:provider]]['exec']
+  if app_config.key?('bootstrap') && config['providers'][provider].key?('exec')
+    remote_command = config['providers'][[provider]]['exec']
     app_config['bootstrap'].each do |cmd|
       commands[:build] << Mustache.render(remote_command, options.merge!(command: cmd)).to_s
     end
   end
 
-  if config['providers'][options[:provider]].key?('reboot')
-    commands[:build] << config['providers'][options[:provider]].fetch('reboot')
+  if config.dig('providers', provider).key?('reboot')
+    commands[:build] << config.dig('providers', provider, 'reboot')
     commands[:build] << 'sleep 30'
   end
 
   commands[:build] << 'curl --retry 5 --retry-delay 5 --retry-max-time 180 --retry-connrefused http://`cat ip.txt`:3000 -v'
 
-  unless options[:collect] == 'off'
-    commands[:collect] << "DATABASE_URL=#{ENV['DATABASE_URL']} ../../bin/client --language #{language} --framework #{framework} #{options[:sieger_options]} -h `cat ip.txt`"
-  end
+  commands[:collect] << "LANGUAGE=#{language} FRAMEWORK=#{framework} DATABASE_URL=#{ENV['DATABASE_URL']} bundle exec rake collect"
 
-  unless options[:clean] == 'off'
-    config['providers'][options[:provider]]['clean'].each do |cmd|
-      commands[:clean] << Mustache.render(cmd, options).to_s
-    end
+  config.dig('providers', provider, 'clean').each do |cmd|
+    commands[:clean] << Mustache.render(cmd, options).to_s
   end
 
   commands
@@ -170,7 +155,7 @@ task :config do
 
     makefile = File.open(File.join(language, framework, MANIFESTS[:build]), 'w')
 
-    commands_for(language, framework, provider: provider, clean: clean, sieger_options: sieger_options, path: path, collect: collect).each do |target, commands|
+    commands_for(language, framework, provider).each do |target, commands|
       makefile.write("#{target}:\n")
       commands.each do |command|
         makefile.write("\t #{command}\n")

Rakefile

@@ -0,0 +1,68 @@
+require "open3"
+require "csv"
+require "etc"
+require "bigdecimal/util"
+
+PIPELINE = {
+  GET: File.join(Dir.pwd, "pipeline.lua"),
+  POST: File.join(Dir.pwd, "pipeline_post.lua"),
+}
+
+def insert(db, framework_id, metric, value, concurrency_level_id)
+  res = db.query("INSERT INTO keys (label) VALUES ($1) ON CONFLICT (label) DO UPDATE SET label = $1 RETURNING id", [metric])
+
+  metric_id = res.first["id"]
+
+  res = db.query("INSERT INTO values (key_id, value) VALUES ($1, $2) RETURNING id", [metric_id, value])
+  value_id = res.first["id"]
+
+  db.query("INSERT INTO metrics (value_id, framework_id, concurrency_id) VALUES ($1, $2, $3)", [value_id, framework_id, concurrency_level_id])
+end
+
+task :collect do
+  threads = ENV.fetch("THREADS") { Etc.nprocessors }
+  duration = ENV.fetch("DURATION") { 10 }
+  language = ENV.fetch("LANGUAGE") { raise "please provide the language" }
+  framework = ENV.fetch("FRAMEWORK") { raise "please provide the target framework" }
+  concurrencies = ENV.fetch("CONCURRENCIES") { "10" }
+  routes = ENV.fetch("ROUTES") { "GET:/" }
+  database = ENV.fetch("DATABASE_URL") { raise "please provide a DATABASE_URL (pg only)" }
+
+  hostname = File.read(File.join(Dir.pwd, language, framework, "ip.txt")).strip
+  `wrk -H 'Connection: keep-alive' -d 5s -c 8 --timeout 8 -t #{threads} http://#{hostname}:3000`
+  `wrk -H 'Connection: keep-alive' -d #{duration}s -c 256 --timeout 8 -t #{threads} http://#{hostname}:3000`
+
+  db = PG.connect(database)
+
+  res = db.query("INSERT INTO languages (label) VALUES ($1) ON CONFLICT (label) DO UPDATE SET label = $1 RETURNING id", [language])
+  language_id = res.first["id"]
+
+  res = db.query("INSERT INTO frameworks (language_id, label) VALUES ($1, $2) ON CONFLICT (language_id, label) DO UPDATE SET label = $2 RETURNING id", [language_id, framework])
+  framework_id = res.first["id"]
+
+  routes.split(",").each do |route|
+    method, uri = route.split(":")
+
+    concurrencies.split(",").each do |concurrency|
+      res = db.query("INSERT INTO concurrencies (level) VALUES ($1) ON CONFLICT (level) DO UPDATE SET level = $1 RETURNING id", [concurrency])
+
+      concurrency_level_id = res.first["id"]
+
+      command = format("wrk -H 'Connection: keep-alive' --connections %<concurrency>s --threads %<threads>s --duration %<duration>s --timeout 1 --script %<pipeline>s http://%<hostname>s:3000", concurrency: concurrency, threads: threads, duration: duration, pipeline: PIPELINE[method.to_sym], hostname: hostname)
+
+      Open3.popen3(command) do |_, _, stderr|
+        lua_output = stderr.read
+
+        info = lua_output.split(",")
+        ["duration_ms", "total_requests", "total_requests_per_s", "total_bytes_received",
+         "socket_connection_errors", "socket_read_errors", "socket_write_errors",
+         "http_errors", "request_timeouts", "minimim_latency", "maximum_latency",
+         "average_latency", "standard_deviation", "percentile_50",
+         "percentile_75", "percentile_90", "percentile_99", "percentile_99.999"].each_with_index do |key, index|
+          insert(db, framework_id, key, info[index].to_d, concurrency_level_id)
+        end
+      end
+    end
+  end
+  db.close
+end