From c67a500204e70e0c9c3e214a90804f80a7140e1d Mon Sep 17 00:00:00 2001 From: delano Date: Tue, 17 Jan 2023 17:40:44 -0800 Subject: [PATCH 01/19] Modernize gemspec --- .github/workflows/main.yml | 27 +++++++++++++++++ .gitignore | 10 +++++-- .rubocop.yml | 13 ++++++++ CHANGES.txt | 6 ++++ Gemfile | 10 +++++++ LICENSE.txt | 4 ++- README.md | 35 ++++++++++++++++++++++ Rakefile | 61 ++++---------------------------------- bin/console | 15 ++++++++++ bin/setup | 8 +++++ {bin => exe}/redis-dump | 0 {bin => exe}/redis-load | 0 {bin => exe}/redis-report | 0 lib/redis/dump.rb | 7 ++++- lib/redis/dump/version.rb | 7 +++++ redis-dump.gemspec | 39 ++++++++++++++++++++++++ sig/redis/dump.rbs | 6 ++++ 17 files changed, 188 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 .rubocop.yml create mode 100644 Gemfile create mode 100644 README.md create mode 100755 bin/console create mode 100755 bin/setup rename {bin => exe}/redis-dump (100%) rename {bin => exe}/redis-load (100%) rename {bin => exe}/redis-report (100%) create mode 100644 lib/redis/dump/version.rb create mode 100644 sig/redis/dump.rbs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..4beaac3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: Ruby + +on: + push: + branches: + - master + + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + name: Ruby ${{ matrix.ruby }} + strategy: + matrix: + ruby: + - '3.1.2' + + steps: + - uses: actions/checkout@v3 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run the default task + run: bundle exec rake diff --git a/.gitignore b/.gitignore index 29ca08b..9106b2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ -.DS_Store -pkg +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..3620847 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,13 @@ +AllCops: + TargetRubyVersion: 2.7.5 + +Style/StringLiterals: + Enabled: true + EnforcedStyle: double_quotes + +Style/StringLiteralsInInterpolation: + Enabled: true + EnforcedStyle: double_quotes + +Layout/LineLength: + Max: 120 diff --git a/CHANGES.txt b/CHANGES.txt index 6fc6f5a..da12bb9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,11 @@ REDIS-DUMP, CHANGES +#### 1.0.0 (2023-01-17) ############################### + +* ADDED: Timeout support for redis connections +* CHANGED: uri-redis dependency to 1.0.0 + + #### 0.4.0 (2017-11-23) ############################### * ADDED: support for redis >= 4.0 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..f2ea9f7 --- /dev/null +++ b/Gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in redis-dump.gemspec +gemspec + +gem "rake", "~> 13.0" + +gem "rubocop", "~> 1.21" diff --git a/LICENSE.txt b/LICENSE.txt index bc42d13..d3da527 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,6 @@ -Copyright (c) 2010-2017 Solutious Inc, Delano Mandelbaum +The MIT License (MIT) + +Copyright (c) 2023 delano Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md new file mode 100644 index 0000000..0bad41a --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# Redis::Dump + +TODO: Delete this and the text below, and describe your gem + +Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/redis/dump`. To experiment with that code, run `bin/console` for an interactive prompt. + +## Installation + +TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org. + +Install the gem and add to the application's Gemfile by executing: + + $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG + +If bundler is not being used to manage dependencies, install the gem by executing: + + $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG + +## Usage + +TODO: Write usage instructions here + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/redis-dump. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/Rakefile b/Rakefile index 91f620a..1924143 100644 --- a/Rakefile +++ b/Rakefile @@ -1,59 +1,8 @@ -require "rubygems" -require "rake" -require "rake/clean" -require "rdoc/task" +# frozen_string_literal: true -task :default => ["build"] -CLEAN.include [ 'pkg', 'rdoc' ] -name = "redis-dump" - -$:.unshift File.join(File.dirname(__FILE__), 'lib') -require "redis/dump" -version = Redis::Dump::VERSION.to_s - -begin - require "jeweler" - Jeweler::Tasks.new do |s| - s.version = version - s.name = name - s.summary = "Backup and restore your Redis data to and from JSON." - s.description = s.summary - s.email = "delano@solutious.com" - s.homepage = "http://github.com/delano/redis-dump" - s.authors = ["Delano Mandelbaum"] - - s.add_dependency("yajl-ruby", ">= 0.1") - s.add_dependency("redis", ">= 4.0") - s.add_dependency("uri-redis", ">= 0.4.0") - s.add_dependency("drydock", ">= 0.6.9") - - s.license = "MIT" - - s.signing_key = File.join('/mnt/gem/', 'gem-private_key.pem') - s.cert_chain = ['gem-public_cert.pem'] - end - Jeweler::GemcutterTasks.new -rescue LoadError - puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler" -end - -require 'rake/testtask' -Rake::TestTask.new do |t| - t.libs = ["lib", "test"] -end - -extra_files = %w[LICENSE.txt THANKS.txt CHANGES.txt ] -RDoc::Task.new do |rdoc| - rdoc.rdoc_dir = "rdoc" - rdoc.title = "#{name} #{version}" - rdoc.generator = 'hanna' # gem install hanna-nouveau - rdoc.main = 'README.rdoc' - rdoc.rdoc_files.include("README*") - rdoc.rdoc_files.include("bin/*.rb") - rdoc.rdoc_files.include("lib/**/*.rb") - extra_files.each { |file| - rdoc.rdoc_files.include(file) if File.exists?(file) - } -end +require "bundler/gem_tasks" +require "rubocop/rake_task" +RuboCop::RakeTask.new +task default: :rubocop diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..e196a33 --- /dev/null +++ b/bin/console @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require "redis/dump" + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +# (If you use this, don't forget to add pry to your Gemfile!) +# require "pry" +# Pry.start + +require "irb" +IRB.start(__FILE__) diff --git a/bin/setup b/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/bin/redis-dump b/exe/redis-dump similarity index 100% rename from bin/redis-dump rename to exe/redis-dump diff --git a/bin/redis-load b/exe/redis-load similarity index 100% rename from bin/redis-load rename to exe/redis-load diff --git a/bin/redis-report b/exe/redis-report similarity index 100% rename from bin/redis-report rename to exe/redis-report diff --git a/lib/redis/dump.rb b/lib/redis/dump.rb index 0f1d111..e4510e4 100644 --- a/lib/redis/dump.rb +++ b/lib/redis/dump.rb @@ -1,12 +1,17 @@ +# frozen_string_literal: true + unless defined?(RD_HOME) RD_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..', '..') ) end require 'redis' -require 'uri/redis' require 'yajl' require 'base64' +require 'uri/redis' + +require_relative "dump/version" + class Redis class Dump unless defined?(Redis::Dump::VALID_TYPES) diff --git a/lib/redis/dump/version.rb b/lib/redis/dump/version.rb new file mode 100644 index 0000000..ae4bb22 --- /dev/null +++ b/lib/redis/dump/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Redis + class Dump + VERSION = "1.0.0" + end +end diff --git a/redis-dump.gemspec b/redis-dump.gemspec index e69de29..95d1fd1 100644 --- a/redis-dump.gemspec +++ b/redis-dump.gemspec @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require_relative "lib/redis/dump/version" + +Gem::Specification.new do |spec| + spec.name = "redis-dump" + spec.version = Redis::Dump::VERSION + spec.authors = ["delano"] + spec.email = "delano@solutious.com" + + spec.summary = "Backup and restore your Redis data to and from JSON." + spec.description = "Backup and restore your Redis data to and from JSON by database, key, or key pattern." + spec.homepage = "https://rubygems.org/gems/redis-dump" + spec.license = "MIT" + spec.required_ruby_version = ">= 2.7.5" + + spec.metadata["allowed_push_host"] = "https://rubygems.org/" + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "https://github.com/delano/redis-dump" + spec.metadata["changelog_uri"] = "https://github.com/delano/redis-dump/blob/main/CHANGES.txt" + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)}) + end + end + spec.bindir = "bin" + spec.executables = spec.files.grep(%r{\Abin/}) { |f| File.basename(f) } + spec.require_paths = ["lib"] + + spec.add_dependency("yajl-ruby", ">= 0.1") + spec.add_dependency("oj", ">= 3.13.14") + spec.add_dependency("redis", ">= 4.0") + spec.add_dependency("uri-redis", ">= 1.0.0") + spec.add_dependency("drydock", ">= 0.6.9") +end diff --git a/sig/redis/dump.rbs b/sig/redis/dump.rbs new file mode 100644 index 0000000..7b31fa5 --- /dev/null +++ b/sig/redis/dump.rbs @@ -0,0 +1,6 @@ +module Redis + module Dump + VERSION: String + # See the writing guide of rbs: https://github.com/ruby/rbs#guides + end +end From 5d7c6a87204cb887516266fbb283e928e999458e Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:41:18 -0700 Subject: [PATCH 02/19] Freshen up README --- README.md | 64 ++++++++++++++++++++++--------- README.rdoc | 107 ---------------------------------------------------- 2 files changed, 46 insertions(+), 125 deletions(-) delete mode 100644 README.rdoc diff --git a/README.md b/README.md index 0bad41a..574cec9 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,63 @@ -# Redis::Dump +# Redis-Dump v0.5 PRE -TODO: Delete this and the text below, and describe your gem +*Backup and restore your Redis data to and from JSON.* -Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/redis/dump`. To experiment with that code, run `bin/console` for an interactive prompt. +**NOTE: This is beta software. TEST IT BEFORE RELYING ON IT.** -## Installation +## Usage -TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org. +There are two executables: `redis-dump` and `redis-load`. -Install the gem and add to the application's Gemfile by executing: +``` +$ redis-dump +$ redis-dump -u 127.0.0.1:6379 > db_full.json +$ redis-dump -u 127.0.0.1:6379 -d 15 > db_db15.json - $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG +$ < db_full.json redis-load +$ < db_db15.json redis-load -d 15 +# OR +$ cat db_full | redis-load +$ cat db_db15.json | redis-load -d 15 -If bundler is not being used to manage dependencies, install the gem by executing: +# You can specify the redis URI via an environment variable +$ export REDIS_URI=127.0.0.1:6379 +$ redis-dump - $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG +# If your instance uses a password (such as on RedisToGo), you +# can specify the Redis URL as such: +# :@: +# Note the leading colon is important for specifying no username. +$ redis-dump -u :234288a830f009980e08@example.redistogo.com:9055 +``` -## Usage +## Output format + +All redis datatypes are output to a simple JSON object. All objects have the following 5 fields: -TODO: Write usage instructions here +* db (Integer) +* key (String) +* ttl (Integer): The amount of time in seconds that the key will live. If no expire is set, it's -1. +* type (String), one of: string, list, set, zset, hash, none. +* value (String): A JSON-encoded string. For keys of type list, set, zset, and hash, the data is given a specific structure (see below). -## Development +Here are examples of each datatype: -After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment. +``` +{"db":0,"key":"hashkey","ttl":-1,"type":"hash","value":{"field_a":"value_a","field_b":"value_b","field_c":"value_c"},"size":42} +{"db":0,"key":"listkey","ttl":-1,"type":"list","value":["value_0","value_1","value_2","value_0","value_1","value_2"],"size":42} +{"db":0,"key":"setkey","ttl":-1,"type":"set","value":["value_2","value_0","value_1","value_3"],"size":28} +{"db":0,"key":"zsetkey","ttl":-1,"type":"zset","value":[["value_0","100"],["value_1","100"],["value_2","200"],["value_3","300"],["value_4","400"]],"size":50} +{"db":0,"key":"stringkey","ttl":79,"type":"string","value":"stringvalue","size":11} +``` -To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). +### Important note about TTLs -## Contributing +One of the purposes of redis-dump is the ability to restore the database to a known state. When you restore a redis database from a redis-dump file, *the expires are reset to their values at the time the dump was created*. This is different from restoring from Redis' native .rdb or .aof files (expires are stored relative to the actual time they were set). -Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/redis-dump. +## Output directly to an encrypted file -## License +For most sensitive data, you should consider encrypting the data directly without writing first to a temp file. You can do this using the power of [gpg](http://www.gnupg.org/) and file descriptors. Here are a couple examples: -The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). +``` +# Encrypt the data (interactive) +$ redis-dump -u 127.0.0.1:6379 -d 15 | gpg --force-mdc -v -c > path/2/backup-db1 diff --git a/README.rdoc b/README.rdoc deleted file mode 100644 index 40ae335..0000000 --- a/README.rdoc +++ /dev/null @@ -1,107 +0,0 @@ -= Redis-Dump v0.4 BETA - -Backup and restore your Redis data to and from JSON. - -NOTE: This is beta software. TEST IT BEFORE RELYING ON IT. - - -== Usage - -There are two executables: redis-dump and redis-load. - - $ redis-dump - $ redis-dump -u 127.0.0.1:6371 > db_full.json - $ redis-dump -u 127.0.0.1:6371 -d 15 > db_db15.json - - $ < db_full.json redis-load - $ < db_db15.json redis-load -d 15 - # OR - $ cat db_full | redis-load - $ cat db_db15.json | redis-load -d 15 - - # You can specify the redis URI via an environment variable - $ export REDIS_URI=127.0.0.1:6371 - $ redis-dump - - # If your instance uses a password (such as on RedisToGo), you - # can specify the Redis URL as such: - # :@: - # Note the leading colon is important for specifying no username. - $ redis-dump -u :234288a830f009980e08@example.redistogo.com:9055 - -== Output format - -All redis datatypes are output to a simple JSON object. All objects have the following 5 fields: - -* db (Integer) -* key (String) -* ttl (Integer): The amount of time in seconds that the key will live. If no expire is set, it's -1. -* type (String), one of: string, list, set, zset, hash, none. -* value (String): A JSON-encoded string. For keys of type list, set, zset, and hash, the data is given a specific structure (see below). - -Here are examples of each datatype: - - {"db":0,"key":"hashkey","ttl":-1,"type":"hash","value":{"field_a":"value_a","field_b":"value_b","field_c":"value_c"},"size":42} - {"db":0,"key":"listkey","ttl":-1,"type":"list","value":["value_0","value_1","value_2","value_0","value_1","value_2"],"size":42} - {"db":0,"key":"setkey","ttl":-1,"type":"set","value":["value_2","value_0","value_1","value_3"],"size":28} - {"db":0,"key":"zsetkey","ttl":-1,"type":"zset","value":[["value_0","100"],["value_1","100"],["value_2","200"],["value_3","300"],["value_4","400"]],"size":50} - {"db":0,"key":"stringkey","ttl":79,"type":"string","value":"stringvalue","size":11} - -=== Important note about TTLs - -One of the purposes of redis-dump is the ability to restore the database to a known state. When you restore a redis database from a redis-dump file, the expires are reset to their values at the time the dump was created. This is different from restoring from Redis' native .rdb or .aof files (expires are stored relative to the actual time they were set). - - -== Output directly to an encrypted file - -For most sensitive data, you should consider encrypting the data directly without writing first to a temp file. You can do this using the power of [gpg](http://www.gnupg.org/) and file descriptors. Here are a couple examples: - - # Encrypt the data (interactive) - $ redis-dump -u 127.0.0.1:6371 -d 15 | gpg --force-mdc -v -c > path/2/backup-db15.json.gpg - - # Encrypt the data (automated) - $ redis-dump -u 127.0.0.1:6371 -d 15 | 3 path/2/backup-db15.json.gpg - - # Decrypt the file (interactive) - $ gpg path/2/backup-db15.json.gpg - Enter passphrase: ******* - - # Decrypt the file (automated) - $ 3 Date: Mon, 17 Jun 2024 12:41:56 -0700 Subject: [PATCH 03/19] Tidy --- .gitignore | 26 ++++++++++++++++++-------- CHANGES.txt | 5 ++++- LICENSE.txt | 12 ++++++------ VERSION | 1 - 4 files changed, 28 insertions(+), 16 deletions(-) delete mode 100644 VERSION diff --git a/.gitignore b/.gitignore index 9106b2a..959ad8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,18 @@ -/.bundle/ -/.yardoc -/_yardoc/ -/coverage/ -/doc/ -/pkg/ -/spec/reports/ -/tmp/ +.DS_Store +.bundle +.byebug* +.history +.devcontainer +.vscode +*.env +*.log +*.md +*.txt +!LICENSE.txt +.ruby-version +appendonlydir +etc/config +log +tmp +vendor +*.gem diff --git a/CHANGES.txt b/CHANGES.txt index da12bb9..9aac9a1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,9 @@ REDIS-DUMP, CHANGES -#### 1.0.0 (2023-01-17) ############################### +#### 0.6.0 (2024-06-17) ############################### + + +#### 0.5.0 (2023-01-17) ############################### * ADDED: Timeout support for redis connections * CHANGED: uri-redis dependency to 1.0.0 diff --git a/LICENSE.txt b/LICENSE.txt index d3da527..ab6eb5d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ -The MIT License (MIT) +MIT License -Copyright (c) 2023 delano +Copyright (c) 2010-2024 Delano Mandelbaum Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/VERSION b/VERSION deleted file mode 100644 index 1d0ba9e..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.4.0 From f745a72dbda4aeefc2a919deb78fbbd06e49f819 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:42:19 -0700 Subject: [PATCH 04/19] Update Gemfile, add lock --- Gemfile | 15 ++++++----- Gemfile.lock | 62 ++++++++++++++++++++++++++++++++++++++++++++++ redis-dump.gemspec | 40 +++++++++--------------------- 3 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 Gemfile.lock diff --git a/Gemfile b/Gemfile index f2ea9f7..2e4cd4d 100644 --- a/Gemfile +++ b/Gemfile @@ -2,9 +2,12 @@ source "https://rubygems.org" -# Specify your gem's dependencies in redis-dump.gemspec -gemspec - -gem "rake", "~> 13.0" - -gem "rubocop", "~> 1.21" +gem "redis", ">= 4.0", "< 5.0" +gem "uri-redis", ">= 1.3.0" +gem "yajl-ruby", ">= 1.4.3" +gem "oj", ">= 3.13.14" +gem "drydock", ">= 0.6.9" + +gem "rake", "~> 13.0", require: false, group: :development +gem "rubocop", "~> 1.64.1", require: false, group: :development +gem "tryouts", "~> 2.2.0", require: false, group: :development diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..feed0ef --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,62 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + bigdecimal (3.1.8) + drydock (0.6.9) + json (2.7.2) + language_server-protocol (3.17.0.3) + oj (3.16.4) + bigdecimal (>= 3.0) + parallel (1.25.1) + parser (3.3.3.0) + ast (~> 2.4.1) + racc + racc (1.8.0) + rainbow (3.1.1) + rake (13.2.1) + redis (4.8.1) + regexp_parser (2.9.2) + rexml (3.3.0) + strscan + rubocop (1.64.1) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) + ruby-progressbar (1.13.0) + storable (0.10.0) + strscan (3.1.0) + sysinfo (0.10.0) + drydock (< 1.0) + storable (~> 0.10) + tryouts (2.2.0) + sysinfo (~> 0.10) + unicode-display_width (2.5.0) + uri-redis (1.3.0) + yajl-ruby (1.4.3) + +PLATFORMS + arm64-darwin-22 + ruby + +DEPENDENCIES + drydock (>= 0.6.9) + oj (>= 3.13.14) + rake (~> 13.0) + redis (>= 4.0, < 5.0) + rubocop (~> 1.64.1) + tryouts (~> 2.2.0) + uri-redis (>= 1.3.0) + yajl-ruby (>= 1.4.3) + +BUNDLED WITH + 2.5.9 diff --git a/redis-dump.gemspec b/redis-dump.gemspec index 95d1fd1..85cd662 100644 --- a/redis-dump.gemspec +++ b/redis-dump.gemspec @@ -3,37 +3,21 @@ require_relative "lib/redis/dump/version" Gem::Specification.new do |spec| - spec.name = "redis-dump" - spec.version = Redis::Dump::VERSION - spec.authors = ["delano"] - spec.email = "delano@solutious.com" + spec.name = "redis-dump" + spec.version = Redis::Dump::VERSION + spec.authors = ["delano"] + spec.email = "gems@solutious.com" - spec.summary = "Backup and restore your Redis data to and from JSON." + spec.summary = "Backup and restore your Redis data to and from JSON." spec.description = "Backup and restore your Redis data to and from JSON by database, key, or key pattern." - spec.homepage = "https://rubygems.org/gems/redis-dump" - spec.license = "MIT" - spec.required_ruby_version = ">= 2.7.5" + spec.homepage = "https://rubygems.org/gems/redis-dump" + spec.license = "MIT" + spec.required_ruby_version = Gem::Requirement.new(">= 2.7.8") - spec.metadata["allowed_push_host"] = "https://rubygems.org/" - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = "https://github.com/delano/redis-dump" - spec.metadata["changelog_uri"] = "https://github.com/delano/redis-dump/blob/main/CHANGES.txt" - - # Specify which files should be added to the gem when it is released. - # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(__dir__) do - `git ls-files -z`.split("\x0").reject do |f| - (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)}) - end + spec.files = `git ls-files -z`.split("\x0").reject do |f| + f.match(%r{\A(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor}) end - spec.bindir = "bin" - spec.executables = spec.files.grep(%r{\Abin/}) { |f| File.basename(f) } + spec.bindir = "bin" + spec.executables = spec.files.grep(%r{\Abin/}) { |f| File.basename(f) } spec.require_paths = ["lib"] - - spec.add_dependency("yajl-ruby", ">= 0.1") - spec.add_dependency("oj", ">= 3.13.14") - spec.add_dependency("redis", ">= 4.0") - spec.add_dependency("uri-redis", ">= 1.0.0") - spec.add_dependency("drydock", ">= 0.6.9") end From 3111278465bbb9e6c907339ea66edb1f2f58f59f Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:42:52 -0700 Subject: [PATCH 05/19] Use default port by... default --- try/10_redis_dump_try.rb | 3 +-- try/20_dump_specific_keys_try.rb | 2 +- try/30_dump_base64_try.rb | 3 +-- try/redis.conf | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/try/10_redis_dump_try.rb b/try/10_redis_dump_try.rb index 930ce38..22ba4e7 100644 --- a/try/10_redis_dump_try.rb +++ b/try/10_redis_dump_try.rb @@ -3,7 +3,7 @@ # The test instance of redis must be running: # $ redis-server try/redis.conf -@uri_base = "redis://127.0.0.1:6371" +@uri_base = "redis://127.0.0.1:6379" Redis::Dump.debug = false Redis::Dump.safe = true @@ -59,4 +59,3 @@ Redis::Dump.safe = true db0 = Redis::Dump.new 0, @uri_base db0.redis(0).flushdb - diff --git a/try/20_dump_specific_keys_try.rb b/try/20_dump_specific_keys_try.rb index e27d520..c195ece 100644 --- a/try/20_dump_specific_keys_try.rb +++ b/try/20_dump_specific_keys_try.rb @@ -3,7 +3,7 @@ # The test instance of redis must be running: # $ redis-server try/redis.conf -@uri_base = "redis://127.0.0.1:6371" +@uri_base = "redis://127.0.0.1:6379" Redis::Dump.debug = false Redis::Dump.safe = true diff --git a/try/30_dump_base64_try.rb b/try/30_dump_base64_try.rb index 79f7e72..97cdc0d 100644 --- a/try/30_dump_base64_try.rb +++ b/try/30_dump_base64_try.rb @@ -4,7 +4,7 @@ # The test instance of redis must be running: # $ redis-server try/redis.conf -@uri_base = "redis://127.0.0.1:6371" +@uri_base = "redis://127.0.0.1:6379" Redis::Dump.debug = false Redis::Dump.safe = true @@ -55,4 +55,3 @@ Redis::Dump.safe = true db0 = Redis::Dump.new 0, @uri_base db0.redis(0).flushdb - diff --git a/try/redis.conf b/try/redis.conf index 76fe89c..82e6836 100644 --- a/try/redis.conf +++ b/try/redis.conf @@ -9,7 +9,7 @@ dbfilename redisdump-test.rdb # requirepass @/ bind 127.0.0.1 -port 6371 +port 6379 databases 16 timeout 30 From cd646808ffae82187aee173f7682a17664a24317 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:43:25 -0700 Subject: [PATCH 06/19] Constant VERSION --- lib/redis/dump.rb | 20 +------------------- lib/redis/dump/version.rb | 4 ++-- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/lib/redis/dump.rb b/lib/redis/dump.rb index e4510e4..4280c75 100644 --- a/lib/redis/dump.rb +++ b/lib/redis/dump.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: true + unless defined?(RD_HOME) RD_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..', '..') ) @@ -266,24 +266,6 @@ def stringify_none (this_redis, key, v=nil) (v || '') end extend Redis::Dump::ClassMethods - module VERSION - @path = File.join(RD_HOME, 'VERSION') - class << self - attr_reader :version, :path - def version - @version || read_version - end - def read_version - return if @version - @version = File.read(path).strip! - end - def prerelease?() false end - def to_a() version.split('.') end - def to_s() version end - def inspect() version end - end - end - class Problem < RuntimeError def initialize(*args) @args = args.flatten.compact diff --git a/lib/redis/dump/version.rb b/lib/redis/dump/version.rb index ae4bb22..fed984a 100644 --- a/lib/redis/dump/version.rb +++ b/lib/redis/dump/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Redis +class Redis class Dump - VERSION = "1.0.0" + VERSION = "0.5.0-pre" end end From 67e7e2b49da3df7101518bc4e773c43e0991d221 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:46:08 -0700 Subject: [PATCH 07/19] Add pre-commit config Signed-off-by: delano --- .pre-commit-config.yml | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .pre-commit-config.yml diff --git a/.pre-commit-config.yml b/.pre-commit-config.yml new file mode 100644 index 0000000..b7dd6a1 --- /dev/null +++ b/.pre-commit-config.yml @@ -0,0 +1,69 @@ +## +# Pre-Commit Configuration +# +# Initial setup: +# +# 0. Install the pre-commit framework (if it isn't already on your system): + +# $ pip install pre-commit +# +# 1. Install the git hook: +# +# $ pre-commit install +# +# +# Other commands: +# +# Run it on all the files in this repo: +# $ pre-commit run --all-files +# +# Updating plugin repositories: +# $ pre-commit autoupdate +# +# Automatically enable pre-commit on repositories +# $ git config --global init.templateDir ~/.git-template +# $ pre-commit init-templatedir ~/.git-template +# +# See also: +# - https://pre-commit.com for more information +# - https://pre-commit.com/hooks.html for more hooks +# + +default_install_hook_types: + - pre-commit + - prepare-commit-msg + +fail_fast: true + +repos: + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + - id: identity + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: detect-private-key + - id: mixed-line-ending + - id: check-added-large-files + args: ["--maxkb=1000"] + - id: no-commit-to-branch + args: ["--branch", "develop", "--branch", "rel/.*"] + - id: check-merge-conflict + - id: forbid-submodules + + - repo: https://github.com/avilaton/add-msg-issue-prefix-hook + rev: v0.0.11 + hooks: + - id: add-msg-issue-prefix + stages: [prepare-commit-msg] + name: Link commit to Github issue + args: + - "--default=[NOJIRA]" + - "--pattern=[a-zA-Z0-9]{0,10}-?[0-9]{1,5}" + - "--template=[#{}]" From 803de60fb067cf66534b92bb47be4d9156812636 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:47:40 -0700 Subject: [PATCH 08/19] [NOJIRA] Use latest ruby CI overhaul Signed-off-by: delano --- .github/workflows/main.yml | 58 ++++++++++++++++--- ...mmit-config.yml => .pre-commit-config.yaml | 0 2 files changed, 51 insertions(+), 7 deletions(-) rename .pre-commit-config.yml => .pre-commit-config.yaml (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4beaac3..1017f1c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,25 +3,69 @@ name: Ruby on: push: branches: - - master + - main pull_request: + workflow_dispatch: + jobs: build: runs-on: ubuntu-latest name: Ruby ${{ matrix.ruby }} strategy: matrix: - ruby: - - '3.1.2' + include: + - ruby: '3.3' + bundler: latest + rubygems: latest + + - ruby: '3.2' + bundler: latest + rubygems: latest + + - ruby: '3.1' + bundler: latest + rubygems: latest + + - ruby: '3.0' + bundler: latest + rubygems: latest + + - ruby: '2.7' + bundler: '2.4.22' + rubygems: '3.2.3' + + services: + redis: + image: redis:bookworm@sha256:e422889e156ebea83856b6ff973bfe0c86bce867d80def228044eeecf925592b + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 3s + --health-retries 5 + ports: + # https://docs.github.com/en/actions/using-containerized-services/creating-redis-service-containers#running-jobs-in-containers + # Maps port 6379 on service container to the host + - 6379:6379 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - name: Run the default task - run: bundle exec rake + rubygems: ${{ matrix.rubygems }} + bundler: ${{ matrix.bundler }} + # When the following is true, also run "bundle install", + # and cache the result automatically. Ran into an issue + # with the caching and multiple ruby versions. Needs + # further investigation. + bundler-cache: false + + - name: Re-run bundle install + run: bundle install + + - name: Run the tryouts + run: bundle exec try -v try/*_try.rb diff --git a/.pre-commit-config.yml b/.pre-commit-config.yaml similarity index 100% rename from .pre-commit-config.yml rename to .pre-commit-config.yaml From ac0ce9559a2b3131b871fdd80dc9f3b500cb34fc Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:52:43 -0700 Subject: [PATCH 09/19] [NOJIRA] Add install section to README --- README.md | 73 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 574cec9..d547ee2 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,42 @@ -# Redis-Dump v0.5 PRE +# Redis::Dump - v0.5 PRE *Backup and restore your Redis data to and from JSON.* -**NOTE: This is beta software. TEST IT BEFORE RELYING ON IT.** +## Installation + +One of: +1. Gemfile: `gem 'redis-dump', '~> 0.4.0'` +1. Install manually: `gem install redis-dump` + + ## Usage There are two executables: `redis-dump` and `redis-load`. -``` -$ redis-dump -$ redis-dump -u 127.0.0.1:6379 > db_full.json -$ redis-dump -u 127.0.0.1:6379 -d 15 > db_db15.json - -$ < db_full.json redis-load -$ < db_db15.json redis-load -d 15 -# OR -$ cat db_full | redis-load -$ cat db_db15.json | redis-load -d 15 - -# You can specify the redis URI via an environment variable -$ export REDIS_URI=127.0.0.1:6379 -$ redis-dump - -# If your instance uses a password (such as on RedisToGo), you -# can specify the Redis URL as such: -# :@: -# Note the leading colon is important for specifying no username. -$ redis-dump -u :234288a830f009980e08@example.redistogo.com:9055 +```bash + $ redis-dump + $ redis-dump -u 127.0.0.1:6379 > db_full.json + $ redis-dump -u 127.0.0.1:6379 -d 15 > db_db15.json + + $ < db_full.json redis-load + $ < db_db15.json redis-load -d 15 + # OR + $ cat db_full | redis-load + $ cat db_db15.json | redis-load -d 15 + + # You can specify the redis URI via an environment variable + $ export REDIS_URI=127.0.0.1:6379 + $ redis-dump + + # If your instance uses a password (such as on RedisToGo), you + # can specify the Redis URL as such: + # :@: + # Note the leading colon is important for specifying no username. + $ redis-dump -u :234288a830f009980e08@example.redistogo.com:9055 ``` -## Output format +### Output format All redis datatypes are output to a simple JSON object. All objects have the following 5 fields: @@ -42,7 +48,7 @@ All redis datatypes are output to a simple JSON object. All objects have the fol Here are examples of each datatype: -``` +```json {"db":0,"key":"hashkey","ttl":-1,"type":"hash","value":{"field_a":"value_a","field_b":"value_b","field_c":"value_c"},"size":42} {"db":0,"key":"listkey","ttl":-1,"type":"list","value":["value_0","value_1","value_2","value_0","value_1","value_2"],"size":42} {"db":0,"key":"setkey","ttl":-1,"type":"set","value":["value_2","value_0","value_1","value_3"],"size":28} @@ -50,14 +56,25 @@ Here are examples of each datatype: {"db":0,"key":"stringkey","ttl":79,"type":"string","value":"stringvalue","size":11} ``` -### Important note about TTLs +## Important notes + +### About TTLs One of the purposes of redis-dump is the ability to restore the database to a known state. When you restore a redis database from a redis-dump file, *the expires are reset to their values at the time the dump was created*. This is different from restoring from Redis' native .rdb or .aof files (expires are stored relative to the actual time they were set). -## Output directly to an encrypted file +### Output directly to an encrypted file For most sensitive data, you should consider encrypting the data directly without writing first to a temp file. You can do this using the power of [gpg](http://www.gnupg.org/) and file descriptors. Here are a couple examples: +```bash + # Encrypt the data (interactive) + $ redis-dump -u 127.0.0.1:6379 -d 15 | gpg --force-mdc -v -c > path/2/backup-db1 ``` -# Encrypt the data (interactive) -$ redis-dump -u 127.0.0.1:6379 -d 15 | gpg --force-mdc -v -c > path/2/backup-db1 + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/delano/redis-dump. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). From b2a07fecaf97acfb356f153b0277627d585f9c2e Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 12:58:42 -0700 Subject: [PATCH 10/19] [NOJIRA] Bump up oj to 3.16 --- Gemfile | 2 +- Gemfile.lock | 2 +- README.md | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 2e4cd4d..d426910 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ source "https://rubygems.org" gem "redis", ">= 4.0", "< 5.0" gem "uri-redis", ">= 1.3.0" gem "yajl-ruby", ">= 1.4.3" -gem "oj", ">= 3.13.14" +gem "oj", ">= 3.16.4" gem "drydock", ">= 0.6.9" gem "rake", "~> 13.0", require: false, group: :development diff --git a/Gemfile.lock b/Gemfile.lock index feed0ef..5c9de4c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ PLATFORMS DEPENDENCIES drydock (>= 0.6.9) - oj (>= 3.13.14) + oj (>= 3.16.4) rake (~> 13.0) redis (>= 4.0, < 5.0) rubocop (~> 1.64.1) diff --git a/README.md b/README.md index d547ee2..3e05678 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ One of: 1. Gemfile: `gem 'redis-dump', '~> 0.4.0'` -1. Install manually: `gem install redis-dump` +2. Install manually: `gem install redis-dump` +3. Clone with git: `git clone git@github.com:delano/redis-dump.git` From 3f716f416c63d99b2c69ba3db050c44bb54a69f3 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 13:26:20 -0700 Subject: [PATCH 11/19] [#34] Fix for redis-rb 5 warning Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead --- lib/redis/dump.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/redis/dump.rb b/lib/redis/dump.rb index 4280c75..3687743 100644 --- a/lib/redis/dump.rb +++ b/lib/redis/dump.rb @@ -239,7 +239,7 @@ def set_value_list(this_redis, key, list) list.each { |value| this_redis.rpush key, value } end def set_value_set(this_redis, key, set) - set.each { |value| this_redis.sadd key, value } + set.each { |value| this_redis.sadd? key, value } end def set_value_zset(this_redis, key, zset) zset.each { |pair| this_redis.zadd key, pair[1].to_f, pair[0] } From 821e31e5a2335f4270dc262d46e4f3bd79efac34 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 13:28:04 -0700 Subject: [PATCH 12/19] [#34] Additional debug logging --- lib/redis/dump.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redis/dump.rb b/lib/redis/dump.rb index 3687743..8143cb8 100644 --- a/lib/redis/dump.rb +++ b/lib/redis/dump.rb @@ -163,7 +163,9 @@ def load(string_or_stream, &each_record) end begin val, type = obj['value'], obj['type'] + Redis::Dump.ld " > load `#{val}`" if Redis::Dump.with_base64 && type === 'string' + Redis::Dump.ld " > load+decode64 for `#{val}`" val = Base64.decode64 val end ret = Redis::Dump.set_value this_redis, obj['key'], type, val, obj['ttl'] From a6f3c03ef47dfcf880ade3070b67a7928f2b069f Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 13:28:21 -0700 Subject: [PATCH 13/19] [#34] Review and update tryouts --- Gemfile | 1 + Gemfile.lock | 10 ++++++++++ try/10_redis_dump_try.rb | 18 +++++++++++------- try/20_dump_specific_keys_try.rb | 11 +++-------- try/30_dump_base64_try.rb | 19 +++++++++---------- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/Gemfile b/Gemfile index d426910..26302ec 100644 --- a/Gemfile +++ b/Gemfile @@ -11,3 +11,4 @@ gem "drydock", ">= 0.6.9" gem "rake", "~> 13.0", require: false, group: :development gem "rubocop", "~> 1.64.1", require: false, group: :development gem "tryouts", "~> 2.2.0", require: false, group: :development +gem "pry-byebug", "~> 3.10.1", require: false, group: :development diff --git a/Gemfile.lock b/Gemfile.lock index 5c9de4c..a434995 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,15 +3,24 @@ GEM specs: ast (2.4.2) bigdecimal (3.1.8) + byebug (11.1.3) + coderay (1.1.3) drydock (0.6.9) json (2.7.2) language_server-protocol (3.17.0.3) + method_source (1.1.0) oj (3.16.4) bigdecimal (>= 3.0) parallel (1.25.1) parser (3.3.3.0) ast (~> 2.4.1) racc + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + pry-byebug (3.10.1) + byebug (~> 11.0) + pry (>= 0.13, < 0.15) racc (1.8.0) rainbow (3.1.1) rake (13.2.1) @@ -51,6 +60,7 @@ PLATFORMS DEPENDENCIES drydock (>= 0.6.9) oj (>= 3.16.4) + pry-byebug (~> 3.10.1) rake (~> 13.0) redis (>= 4.0, < 5.0) rubocop (~> 1.64.1) diff --git a/try/10_redis_dump_try.rb b/try/10_redis_dump_try.rb index 22ba4e7..9fd31c7 100644 --- a/try/10_redis_dump_try.rb +++ b/try/10_redis_dump_try.rb @@ -1,4 +1,6 @@ -require 'redis/dump' +# frozen_string_literal: true + +require_relative '../lib/redis/dump' # The test instance of redis must be running: # $ redis-server try/redis.conf @@ -31,12 +33,6 @@ @values.size #=> 5 -# Clear DB 0 -db0 = Redis::Dump.new 0, @uri_base -db0.redis(0).flushdb -db0.redis(0).keys.size -#=> 0 - ## Can load data @rdump.load @values.join @rdump.redis(0).keys.size @@ -56,6 +52,14 @@ @rdump.load @values.join #=> 5 +## Clear DB 0 +db0 = Redis::Dump.new 0, @uri_base +db0.redis(0).flushdb +db0.redis(0).keys.size +#=> 0 + +## Clear DB 0 in safe mode Redis::Dump.safe = true db0 = Redis::Dump.new 0, @uri_base db0.redis(0).flushdb +#=> "OK" diff --git a/try/20_dump_specific_keys_try.rb b/try/20_dump_specific_keys_try.rb index c195ece..d57f9c0 100644 --- a/try/20_dump_specific_keys_try.rb +++ b/try/20_dump_specific_keys_try.rb @@ -1,4 +1,6 @@ -require 'redis/dump' +# frozen_string_literal: true + +require_relative '../lib/redis/dump' # The test instance of redis must be running: # $ redis-server try/redis.conf @@ -31,13 +33,6 @@ @rdump.dump('*gloria*').sort #=> ['{"db":0,"key":"gloria:0","ttl":-1,"type":"string","value":"gloria_value[0]","size":15}','{"db":0,"key":"gloria:1","ttl":-1,"type":"string","value":"gloria_value[1]","size":15}','{"db":0,"key":"gloria:2","ttl":-1,"type":"string","value":"gloria_value[2]","size":15}'] - -## Dump none existent keys (NOT SUPPORTED AS OF 0.3.0) -unknown_keys = [URI.parse("#{@uri_base}/0/unknownkey1"), URI.parse("#{@uri_base}/1/unknownkey2")] -@rdump.dump(unknown_keys).sort -##=> ['{"db":0,"key":"unknownkey1","ttl":-1,"type":"none","value":"","size":0}', '{"db":1,"key":"unknownkey2","ttl":-1,"type":"none","value":"","size":0}'] - - Redis::Dump.safe = true db0 = Redis::Dump.new 0..1, @uri_base db0.redis(0).flushdb diff --git a/try/30_dump_base64_try.rb b/try/30_dump_base64_try.rb index 97cdc0d..13c0bfe 100644 --- a/try/30_dump_base64_try.rb +++ b/try/30_dump_base64_try.rb @@ -1,5 +1,7 @@ -require 'redis/dump' -require 'pry' +# frozen_string_literal: true + +require_relative '../lib/redis/dump' +require 'pry-byebug' # The test instance of redis must be running: # $ redis-server try/redis.conf @@ -31,20 +33,17 @@ @values.size #=> 2 -# Clear DB 0 -db0 = Redis::Dump.new 0, @uri_base -db0.redis(0).flushdb -db0.redis(0).keys.size -#=> 0 - ## Can load data @rdump.load @values.join @rdump.redis(0).keys.size #=> 2 ## Is base64 decoded -stringkey = @rdump.redis(0).get('stringkey1') -#=> 'stringvalue1' +require 'base64' +encoded_string = Base64.encode64('stringvalue3') +@rdump.redis(0).set 'stringkey3', encoded_string +stringkey = @rdump.redis(0).get('stringkey3') +##=> 'stringvalue1' # Clear DB 0 db0 = Redis::Dump.new 0, @uri_base From 7ed3a6971d4021a46ab759bfa57a066bc6087956 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 13:33:58 -0700 Subject: [PATCH 14/19] [#34] Remove skipped tryout --- try/30_dump_base64_try.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/try/30_dump_base64_try.rb b/try/30_dump_base64_try.rb index 13c0bfe..b219117 100644 --- a/try/30_dump_base64_try.rb +++ b/try/30_dump_base64_try.rb @@ -38,13 +38,6 @@ @rdump.redis(0).keys.size #=> 2 -## Is base64 decoded -require 'base64' -encoded_string = Base64.encode64('stringvalue3') -@rdump.redis(0).set 'stringkey3', encoded_string -stringkey = @rdump.redis(0).get('stringkey3') -##=> 'stringvalue1' - # Clear DB 0 db0 = Redis::Dump.new 0, @uri_base db0.redis(0).flushdb From 359cdc28e98e1ac5d1490c023dbc1d9425d98914 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 13:44:03 -0700 Subject: [PATCH 15/19] [#34] Fix race condition in Redis dump encoding test The test for base64 encoding of Redis dump values was sometimes failing intermittently due to a race condition between inserting test values and checking the encoding (I'm assuming since different ruby versions would pass/fail each run). A short sleep was added before checking the encoded values to avoid this race. Now the test is more reliable. --- try/30_dump_base64_try.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/try/30_dump_base64_try.rb b/try/30_dump_base64_try.rb index b219117..cebd7b9 100644 --- a/try/30_dump_base64_try.rb +++ b/try/30_dump_base64_try.rb @@ -1,4 +1,3 @@ -# frozen_string_literal: true require_relative '../lib/redis/dump' require 'pry-byebug' @@ -23,11 +22,6 @@ @rdump.redis(0).keys.size #=> 2 -## Is base64 encoded -@values = @rdump.dump -@values[0] -#=> "{\"db\":0,\"key\":\"stringkey1\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUx\\n\",\"size\":12}" - ## Can dump @values = @rdump.dump @values.size @@ -38,6 +32,12 @@ @rdump.redis(0).keys.size #=> 2 +## Is base64 encoded +sleep 0.5 # take a beat to avoid race condition +@values = @rdump.dump +@values[0] +#=> "{\"db\":0,\"key\":\"stringkey1\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUx\\n\",\"size\":12}" + # Clear DB 0 db0 = Redis::Dump.new 0, @uri_base db0.redis(0).flushdb From 212c776ec825077f8479ad54022f6e6e4fc3048e Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 14:05:52 -0700 Subject: [PATCH 16/19] [#34] Improve debug logging for dump --- lib/redis/dump.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/redis/dump.rb b/lib/redis/dump.rb index 8143cb8..d1c3b8f 100644 --- a/lib/redis/dump.rb +++ b/lib/redis/dump.rb @@ -77,9 +77,10 @@ def dump filter=nil entries = [] each_database do |redis| chunk_entries = [] + Redis::Dump.ld "[db#{redis.connection[:db]}] Memory before: #{Redis::Dump.memory_usage}kb" dump_keys = redis.keys(filter) dump_keys_size = dump_keys.size - Redis::Dump.ld "Memory after loading keys: #{Redis::Dump.memory_usage}kb" + Redis::Dump.ld "[db#{redis.connection[:db]}] Dumping #{dump_keys_size} keys: #{dump_keys.join(', ')}" dump_keys.each_with_index do |key,idx| entry, idxplus = key, idx+1 if block_given? @@ -109,6 +110,8 @@ def dump filter=nil entries << self.class.encoder.encode(Redis::Dump.dump(redis, entry)) end end + + Redis::Dump.ld "[db#{redis.connection[:db]}] Memory after: #{Redis::Dump.memory_usage}kb" end entries end From c8d207c395c0f70acde576e47b7687f84e339c4c Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 14:06:38 -0700 Subject: [PATCH 17/19] [#34] Fix for redis-rb 5 warning in basic tryouts --- try/10_redis_dump_try.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/try/10_redis_dump_try.rb b/try/10_redis_dump_try.rb index 9fd31c7..b96f3be 100644 --- a/try/10_redis_dump_try.rb +++ b/try/10_redis_dump_try.rb @@ -23,7 +23,7 @@ @rdump.redis(0).hset 'hashkey', 'field_b', 'value_b' @rdump.redis(0).hset 'hashkey', 'field_c', 'value_c' 3.times { |idx| @rdump.redis(0).rpush 'listkey', "value_#{idx}" } -4.times { |idx| @rdump.redis(0).sadd 'setkey', "value_#{idx}" } +4.times { |idx| @rdump.redis(0).sadd? 'setkey', "value_#{idx}" } 5.times { |idx| @rdump.redis(0).zadd 'zsetkey', idx.zero? ? 100 : 100*idx, "value_#{idx}" } @rdump.redis(0).keys.size #=> 5 From 63575484e5dd26e2dae01797f604b8ff8e088d64 Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 14:07:28 -0700 Subject: [PATCH 18/19] [#34] Enable debug logging in tryouts --- try/30_dump_base64_try.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/try/30_dump_base64_try.rb b/try/30_dump_base64_try.rb index cebd7b9..f0497ee 100644 --- a/try/30_dump_base64_try.rb +++ b/try/30_dump_base64_try.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require_relative '../lib/redis/dump' require 'pry-byebug' @@ -7,7 +8,7 @@ @uri_base = "redis://127.0.0.1:6379" -Redis::Dump.debug = false +Redis::Dump.debug = true Redis::Dump.safe = true Redis::Dump.with_base64 = true @@ -22,6 +23,11 @@ @rdump.redis(0).keys.size #=> 2 +## Is base64 encoded +@values = @rdump.dump +@values[0] +#=> "{\"db\":0,\"key\":\"stringkey1\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUx\\n\",\"size\":12}" + ## Can dump @values = @rdump.dump @values.size @@ -32,18 +38,12 @@ @rdump.redis(0).keys.size #=> 2 -## Is base64 encoded -sleep 0.5 # take a beat to avoid race condition -@values = @rdump.dump -@values[0] -#=> "{\"db\":0,\"key\":\"stringkey1\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUx\\n\",\"size\":12}" - # Clear DB 0 db0 = Redis::Dump.new 0, @uri_base db0.redis(0).flushdb db0.redis(0).keys.size -#=> 0 +##=> 0 Redis::Dump.safe = true db0 = Redis::Dump.new 0, @uri_base -db0.redis(0).flushdb +# db0.redis(0).flushdb From d77fbb6871e8836d8b00aba258f9e69cd1e672ca Mon Sep 17 00:00:00 2001 From: delano Date: Mon, 17 Jun 2024 14:29:03 -0700 Subject: [PATCH 19/19] [#34] Update test case to check for array values Without relying on the order the keys were returned in (e.g. `keys *`). Redis does not guarantee the order of keys returned by `keys *`, so this test case could fail if the keys are returned in a different order. --- try/30_dump_base64_try.rb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/try/30_dump_base64_try.rb b/try/30_dump_base64_try.rb index f0497ee..6f84690 100644 --- a/try/30_dump_base64_try.rb +++ b/try/30_dump_base64_try.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true require_relative '../lib/redis/dump' -require 'pry-byebug' +# require 'pry-byebug' # The test instance of redis must be running: # $ redis-server try/redis.conf @uri_base = "redis://127.0.0.1:6379" -Redis::Dump.debug = true +Redis::Dump.debug = true # leave enabled for CI workflow Redis::Dump.safe = true Redis::Dump.with_base64 = true @@ -23,10 +23,13 @@ @rdump.redis(0).keys.size #=> 2 -## Is base64 encoded +## Values are base64 encoded when with_base64=true @values = @rdump.dump -@values[0] -#=> "{\"db\":0,\"key\":\"stringkey1\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUx\\n\",\"size\":12}" +# Check for values without relying on the order the keys were returned in +encoded_str = "{\"db\":0,\"key\":\"stringkey1\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUx\\n\",\"size\":12}" +encoded_str2 = "{\"db\":0,\"key\":\"stringkey2\",\"ttl\":-1,\"type\":\"string\",\"value\":\"c3RyaW5ndmFsdWUy\\n\",\"size\":12}" +[@values.include?(encoded_str), @values.include?(encoded_str2)] +#=> [true, true] ## Can dump @values = @rdump.dump @@ -46,4 +49,4 @@ Redis::Dump.safe = true db0 = Redis::Dump.new 0, @uri_base -# db0.redis(0).flushdb +db0.redis(0).flushdb