initial commit

Author: Abdul Dakkak

.travis.yml

@@ -0,0 +1,25 @@
+language: go
+matrix:
+  include:
+    - go: 1.7.x
+    - go: 1.8.x
+    - go: tip
+  allow_failures:
+    - go: tip
+dist: trusty
+sudo: false
+addons:
+  apt:
+    sources:
+    - sourceline: ppa:masterminds/glide
+    packages:
+    - glide
+install:
+  - glide install --force
+  - rm -fr vendor/github.com/Sirupsen
+  - find vendor -type f -exec sed -i 's/Sirupsen/sirupsen/g' {} +
+  - go build
+script:
+  - echo "Skip tests..."
+after_script:
+  - go test -race -v $(glide novendor)

config.go

@@ -0,0 +1,52 @@
+package dldataset
+
+import (
+	"github.com/k0kubun/pp"
+	"github.com/rai-project/config"
+	"github.com/rai-project/vipertags"
+)
+
+type dldatasetConfig struct {
+	WorkingDirectory string        `json:"working_directory" config:"dldataset.working_directory" default:""`
+	done             chan struct{} `json:"-" config:"-"`
+}
+
+var (
+	// Config holds the data read by rai-project/config
+	Config = &dldatasetConfig{
+		done: make(chan struct{}),
+	}
+)
+
+func (dldatasetConfig) ConfigName() string {
+	return "DLDataset"
+}
+
+func (c *dldatasetConfig) SetDefaults() {
+	vipertags.SetDefaults(c)
+}
+
+func (c *dldatasetConfig) Read() {
+	defer close(c.done)
+	config.App.Wait()
+	vipertags.Fill(c)
+	if c.WorkingDirectory == "" || c.WorkingDirectory == "default" {
+		c.WorkingDirectory = config.App.TempDir
+	}
+}
+
+func (c dldatasetConfig) Wait() {
+	<-c.done
+}
+
+func (c dldatasetConfig) String() string {
+	return pp.Sprintln(c)
+}
+
+func (c dldatasetConfig) Debug() {
+	log.Debug("DLDataset Config = ", c)
+}
+
+func init() {
+	config.Register(Config)
+}

dldataset.go

@@ -0,0 +1,23 @@
+package dldataset
+
+import (
+	"io"
+
+	context "golang.org/x/net/context"
+)
+
+type LabeledData interface {
+	Label() string
+	Data() (io.Reader, error)
+}
+
+type Dataset interface {
+	New(ctx context.Context) (Dataset, error)
+	Category() string
+	Name() string
+	CanonicalName() string
+	Download(ctx context.Context) error
+	List(ctx context.Context) ([]string, error)
+	Get(ctx context.Context, name string) (LabeledData, error)
+	io.Closer
+}

glide.yaml

@@ -0,0 +1,24 @@
+package: github.com/rai-project/dldataset
+import:
+- package: github.com/Unknwon/com
+  version: ~1.0.0
+- package: github.com/k0kubun/pp
+  version: ~2.3.0
+- package: github.com/pkg/errors
+  version: ~0.8.0
+- package: github.com/rai-project/config
+- package: github.com/rai-project/downloadmanager
+- package: github.com/rai-project/logger
+- package: github.com/rai-project/vipertags
+- package: github.com/sirupsen/logrus
+  version: ^1.0.0
+- package: golang.org/x/net
+  subpackages:
+  - context
+- package: golang.org/x/sync
+  subpackages:
+  - syncmap
+- package: github.com/stretchr/testify
+  version: ~1.1.4
+  subpackages:
+  - assert

init.go

@@ -0,0 +1,17 @@
+package dldataset
+
+import (
+	"github.com/rai-project/config"
+	"github.com/rai-project/logger"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	log *logrus.Entry
+)
+
+func init() {
+	config.AfterInit(func() {
+		log = logger.New().WithField("pkg", "dldataset")
+	})
+}

register.go

@@ -0,0 +1,47 @@
+package dldataset
+
+import (
+	"errors"
+	"path"
+	"strings"
+
+	"golang.org/x/sync/syncmap"
+)
+
+var datasets syncmap.Map
+
+func Get(category, name string) (Dataset, error) {
+	category = strings.ToLower(category)
+	name = strings.ToLower(name)
+	key := path.Join(category, name)
+	val, ok := datasets.Load(key)
+	if !ok {
+		log.WithField("category", category).
+			WithField("name", name).
+			Warn("cannot find dataset")
+		return nil, errors.New("cannot find dataset")
+	}
+	dataset, ok := val.(Dataset)
+	if !ok {
+		log.WithField("category", category).
+			WithField("name", name).
+			Warn("invalid dataset")
+		return nil, errors.New("invalid dataset")
+	}
+	return dataset, nil
+}
+
+func Register(d Dataset) {
+	datasets.Store(d.CanonicalName(), d)
+}
+
+func Datasets() []string {
+	names := []string{}
+	datasets.Range(func(key, _ interface{}) bool {
+		if name, ok := key.(string); ok {
+			names = append(names, name)
+		}
+		return true
+	})
+	return names
+}

vision/base.go

@@ -0,0 +1,30 @@
+package vision
+
+import (
+	"bytes"
+	"io"
+
+	context "golang.org/x/net/context"
+)
+
+type base struct {
+	ctx            context.Context
+	baseWorkingDir string
+}
+
+func (base) Category() string {
+	return "vision"
+}
+
+type LabeledImage struct {
+	label string
+	data  []byte
+}
+
+func (l LabeledImage) Label() string {
+	return l.label
+}
+
+func (l LabeledImage) Data() (io.Reader, error) {
+	return bytes.NewBuffer(l.data), nil
+}

vision/checksum.go

@@ -0,0 +1,37 @@
+package vision
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+type md5sumTy struct{}
+
+var md5sum = md5sumTy{}
+
+func (md5sumTy) Check(reader io.Reader, expected string) (bool, error) {
+	hash := md5.New()
+	_, err := io.Copy(hash, reader)
+	if err != nil {
+		return false, errors.Wrap(err, "failed to copy reader to md5 hash")
+	}
+	actual := hex.EncodeToString(hash.Sum(nil))
+	return actual == expected, nil
+}
+
+func (md5sumTy) CheckFile(path string, expected string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to open %s while performing md5 checksum", path)
+	}
+	defer f.Close()
+	ok, err := md5sum.Check(f, md5)
+	if err != nil {
+		return false, errors.Wrapf(err, "unable to perform md5sum on %s", path)
+	}
+	return ok, nil
+}