First working version (#4)

.circleci/config.yml

@@ -3,11 +3,12 @@ jobs:
   build:
     working_directory: /go/src/github.com/31z4/harvest
     docker:
-      - image: golang:1.8
+      - image: golang:1
+      - image: redis:4-alpine
     steps:
       - checkout
 
       - run: go get github.com/mattn/goveralls
 
-      - run: go test -v -cover -race -coverprofile=coverage.out ./...
+      - run: make test
       - run: goveralls -coverprofile=coverage.out -service=circle-ci

.dockerignore

@@ -0,0 +1,6 @@
+*
+
+!vendor
+!Gopkg.*
+!*.go
+!Makefile

.gitignore

@@ -0,0 +1,3 @@
+build
+release
+*.out

CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [email protected]. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1 as builder
+
+COPY . src/github.com/31z4/harvest
+RUN cd src/github.com/31z4/harvest && make release
+
+
+FROM scratch
+LABEL maintainer="Elisey Zanko <[email protected]>"
+
+ENV COLUMNS=80
+ENTRYPOINT ["./harvest"]
+COPY --from=builder /go/src/github.com/31z4/harvest/release/harvest .

Gopkg.toml

@@ -0,0 +1,5 @@
+[[constraint]]
+  name = "github.com/go-redis/redis"
+
+[[constraint]]
+  name = "gopkg.in/alecthomas/kingpin.v2"

Makefile

@@ -0,0 +1,22 @@
+PACKAGE := github.com/31z4/harvest
+BUILD_DIR := build
+RELEASE_DIR := release
+
+build:
+	go build -o $(BUILD_DIR)/harvest $(PACKAGE)
+
+release:
+	CGO_ENABLED=0 go build -ldflags '-s' -o $(RELEASE_DIR)/harvest $(PACKAGE)
+
+docker:
+	docker build --force-rm -t harvest .
+
+test:
+	go test -tags test -v -race -coverprofile=coverage.out $(PACKAGE)
+
+clean:
+	$(RM) *.out
+	$(RM) -r $(BUILD_DIR)
+	$(RM) -r $(RELEASE_DIR)
+
+.PHONY: test build release docker clean

README.md

@@ -4,4 +4,47 @@
 
 # Harvest
 
-Harvest helps you understand what's inside your Redis.
+`harvest` helps you understand what's inside your Redis. It samples Redis keys and returns some insightful stats. It particular it aims to answer two simple questions that are hard to answer using existing tools. The questions are:
+
+1. What are top 10 key prefixes?
+2. What are top 10 most memory consuming keys?
+
+Answers to these questions should help you figure out why your Redis has suddenly started to eat so much memory or even reached its capacity.
+
+The tool is designed to be simple and efficient. It does not require Redis [`DEBUG OBJECT`](https://redis.io/commands/debug-object) command to be available. Which is good if you're using AWS ElastiCache.
+
+## Usage
+
+Here is an example of how to use `harvest` with Docker and understand its output. Assuming Redis container name is `redis` and it listens on a standard port `6379`.
+
+```console
+$ docker run --link redis:redis -it --rm 31z4/harvest redis://redis
+warning: database size (12) is less than the number of samples (1000)
+
+_kombu.binding.: 33.98% (648)
+unacked: 9.49% (181)
+_kombu.binding.celery: 9.18% (175)
+_kombu.binding.reply.celery.pidbox: 6.45% (123)
+test: 5.98% (114)
+_kombu.binding.schedule: 5.35% (102)
+_kombu.binding.blocks: 5.19% (99)
+_kombu.binding.default: 4.82% (92)
+_kombu.binding.celery.pidbox: 4.72% (90)
+_kombu.binding.celeryev: 4.46% (85)
+```
+
+The first line of the output warns us that the number of samples is greater than the database size. Which basically means that results may be more inaccurate.
+
+The second line tells that from `1000` randomly sampled keys `648` start from `_kombu.binding.` and it's `33.98%` of all samples. The rest of the lines follow same pattern.
+
+## Feedback
+
+Feedback is greatly appreciated. At this stage, the maintainers are most interested in feedback centered on the user experience (UX) of the tool. Do you have cases that the tool supports well, or doesn't support at all? Do any of the commands have surprising effects, output, or results?
+
+## Contributing
+
+Contributions are greatly appreciated. The maintainers actively manage the issues list, and try to highlight issues suitable for newcomers. The project follows the typical GitHub pull request model. Before starting any work, please either comment on an existing issue, or file a new one.
+
+## License
+
+This project is licensed under the GPLv3 License - see the [LICENSE.md](LICENSE.md) file for details.

main.go

@@ -0,0 +1,29 @@
+//+build !test
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+func main() {
+	kingpin.Version("0.1.0")
+	kingpin.CommandLine.Help = "harvest helps you understand what's inside your Redis."
+
+	samples := kingpin.Flag("samples", "Maximum number of samples.").Short('s').Default("1000").Int()
+	results := kingpin.Flag("results", "Maximum number of output results.").Short('n').Default("10").Int()
+	redisUrl := kingpin.Arg("url", "Redis URL.").Default("redis://localhost").String()
+
+	kingpin.Parse()
+
+	output, err := Sample(*redisUrl, *samples, *results)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	fmt.Println(output)
+}

sampler.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"errors"
+
+	"fmt"
+	"os"
+
+	"github.com/go-redis/redis"
+)
+
+// Sample samples Redis keys and returns statistics about it.
+func Sample(redisUrl string, samples, results int) (string, error) {
+	if samples <= 0 {
+		return "", errors.New("number of samples must be > 0")
+	}
+
+	if results <= 0 {
+		return "", errors.New("number of results must be > 0")
+	}
+
+	opt, err := redis.ParseURL(redisUrl)
+	if err != nil {
+		return "", err
+	}
+
+	client := redis.NewClient(opt)
+
+	var dbSize int64
+	dbSize, err = client.DBSize().Result()
+	if err != nil {
+		return "", err
+	}
+
+	if dbSize == 0 {
+		return "", errors.New("the database is empty")
+	}
+
+	if dbSize < int64(samples) {
+		fmt.Fprintf(os.Stderr, "warning: database size (%v) is less than the number of samples (%v)\n\n", dbSize, samples)
+	}
+
+	trie := NewTrie()
+
+	for i := 0; i < samples; i++ {
+		key, err := client.RandomKey().Result()
+		if err != nil {
+			return "", err
+		}
+
+		trie.Insert(key)
+	}
+
+	return trie.Sprint(results), nil
+}

sampler_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	"fmt"
+
+	"github.com/go-redis/redis"
+)
+
+var client = redis.NewClient(&redis.Options{Addr: "localhost:6379"})
+
+func TestSampleArguments(t *testing.T) {
+	cases := []struct {
+		redisUrl string
+		samples  int
+		results  int
+		err      string
+	}{
+		{
+			"redis://localhost",
+			0,
+			10,
+			"number of samples must be > 0",
+		},
+		{
+			"redis://localhost",
+			10,
+			0,
+			"number of results must be > 0",
+		},
+		{
+			"localhost",
+			1,
+			1,
+			"invalid redis URL scheme: ",
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			_, err := Sample(c.redisUrl, c.samples, c.results)
+			if err.Error() != c.err {
+				t.Errorf("expected: %#v\nresult: %#v", c.err, err)
+			}
+		})
+	}
+}
+
+func TestSample(t *testing.T) {
+	output, err := Sample("redis://localhost:6379", 1, 1)
+	if err == nil {
+		t.Errorf("expected an error but got: %#v", output)
+	}
+
+	err = client.Set("key", "value", 0).Err()
+	if err != nil {
+		t.Errorf("SET: error: %#v\n", err)
+	}
+
+	output, err = Sample("redis://localhost:6379", 10, 1)
+	if err != nil {
+		t.Errorf("unexpected error: %#v", err)
+	}
+
+	if output != "key: 100.00% (10)" {
+		t.Errorf("unexpected output: %#v", output)
+	}
+}
+
+func TestMain(m *testing.M) {
+	dbSize, err := client.DBSize().Result()
+	if err != nil {
+		fmt.Printf("DBSIZE: error: %#v\n", err)
+		os.Exit(1)
+	}
+
+	if dbSize > 0 {
+		fmt.Println("redis database is not empty")
+		os.Exit(1)
+	}
+
+	returnCode := m.Run()
+
+	err = client.FlushDB().Err()
+	if err != nil {
+		fmt.Printf("FLUSHDB: error: %#v\n", err)
+		os.Exit(1)
+	}
+
+	os.Exit(returnCode)
+}