Skip to content

Commit f59de1d

Browse files
author
Abdul Dakkak
committed
initial
1 parent ad65d5c commit f59de1d

File tree

4 files changed

+178
-172
lines changed

4 files changed

+178
-172
lines changed

‎vision/caltech256.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package vision

‎vision/ilsvrc2012_validation.go‎

Lines changed: 1 addition & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -1,175 +1,5 @@
11
package vision
22

33
import (
4-
"net/http"
5-
"os"
6-
"path"
7-
"path/filepath"
8-
"strings"
9-
10-
"github.com/pkg/errors"
11-
"github.com/rai-project/config"
12-
"github.com/rai-project/dldataset"
13-
"github.com/rai-project/downloadmanager"
14-
"github.com/rai-project/image"
15-
"github.com/rai-project/image/types"
16-
context "golang.org/x/net/context"
4+
_ "github.com/PaddlePaddle/recordio"
175
)
18-
19-
var iLSVRC2012Validation *ILSVRC2012Validation
20-
21-
// ILSVRC2012Validation ...
22-
type ILSVRC2012Validation struct {
23-
base
24-
baseURL string
25-
filePaths []string
26-
fileURLs map[string]string
27-
data map[string]ILSVRC2012ValidationLabeledImage
28-
}
29-
30-
// ILSVRC2012ValidationLabeledImage ...
31-
type ILSVRC2012ValidationLabeledImage struct {
32-
label string
33-
data *types.RGBImage
34-
}
35-
36-
// Label ...
37-
func (l ILSVRC2012ValidationLabeledImage) Label() string {
38-
return l.label
39-
}
40-
41-
// Data ...
42-
func (l ILSVRC2012ValidationLabeledImage) Data() (interface{}, error) {
43-
return l.data, nil
44-
}
45-
46-
// New ...
47-
func (d *ILSVRC2012Validation) New(ctx context.Context) (dldataset.Dataset, error) {
48-
return iLSVRC2012Validation, nil
49-
}
50-
51-
func (d *ILSVRC2012Validation) workingDir() string {
52-
category := strings.ToLower(d.Category())
53-
name := strings.ToLower(d.Name())
54-
return filepath.Join(d.baseWorkingDir, category, name)
55-
}
56-
57-
// Name ...
58-
func (d *ILSVRC2012Validation) Name() string {
59-
return "ilsvrc2012_validation"
60-
}
61-
62-
// CanonicalName ...
63-
func (d *ILSVRC2012Validation) CanonicalName() string {
64-
category := strings.ToLower(d.Category())
65-
name := strings.ToLower(d.Name())
66-
key := path.Join(category, name)
67-
return key
68-
}
69-
70-
// Download ...
71-
func (d *ILSVRC2012Validation) Download(ctx context.Context) error {
72-
return nil
73-
}
74-
75-
// List ...
76-
func (d *ILSVRC2012Validation) List(ctx context.Context) ([]string, error) {
77-
return d.filePaths, nil
78-
}
79-
80-
// GetWithoutDownloadManager ...
81-
func (d *ILSVRC2012Validation) GetWithoutDownloadManager(ctx context.Context, name string) (dldataset.LabeledData, error) {
82-
fileURL, ok := d.fileURLs[name]
83-
if !ok {
84-
return nil, errors.Errorf("the file path %v for the dataset %v was not found", name, d.CanonicalName())
85-
}
86-
req, err := http.Get(fileURL)
87-
if err != nil {
88-
return nil, errors.Wrapf(err, "failed to perform http get request to %v", fileURL)
89-
}
90-
defer req.Body.Close()
91-
92-
img, err := image.Read(req.Body)
93-
if err != nil {
94-
return nil, errors.Wrapf(err, "failed to read image from %v", fileURL)
95-
}
96-
97-
if _, ok := img.(*types.RGBImage); !ok {
98-
return nil, errors.Wrapf(err, "failed to read rgb image from %v", fileURL)
99-
}
100-
101-
label := path.Dir(name)
102-
103-
return &ILSVRC2012ValidationLabeledImage{
104-
data: img.(*types.RGBImage),
105-
label: label,
106-
}, nil
107-
}
108-
109-
// Get ...
110-
func (d *ILSVRC2012Validation) Get(ctx context.Context, name string) (dldataset.LabeledData, error) {
111-
fileURL, ok := d.fileURLs[name]
112-
if !ok {
113-
return nil, errors.Errorf("the file path %v for the dataset %v was not found", name, d.CanonicalName())
114-
}
115-
116-
workingDir := d.workingDir()
117-
downloadedFileName := filepath.Join(workingDir, name)
118-
downloadedFileName, err := downloadmanager.DownloadFile(fileURL, downloadedFileName, downloadmanager.Context(ctx))
119-
if err != nil {
120-
return nil, errors.Wrapf(err, "failed to download %v", fileURL)
121-
}
122-
123-
f, err := os.Open(downloadedFileName)
124-
if err != nil {
125-
return nil, errors.Wrapf(err, "failed to open %v", downloadedFileName)
126-
}
127-
128-
defer f.Close()
129-
130-
img, err := image.Read(f)
131-
if err != nil {
132-
return nil, errors.Wrapf(err, "failed to read image from %v", fileURL)
133-
}
134-
135-
if _, ok := img.(*types.RGBImage); !ok {
136-
return nil, errors.Wrapf(err, "failed to read rgb image from %v", fileURL)
137-
}
138-
139-
label := path.Dir(name)
140-
141-
return &ILSVRC2012ValidationLabeledImage{
142-
data: img.(*types.RGBImage),
143-
label: label,
144-
}, nil
145-
}
146-
147-
// Close ...
148-
func (d *ILSVRC2012Validation) Close() error {
149-
return nil
150-
}
151-
152-
func init() {
153-
const fileListPath = "/vision/support/ilsvrc2012_validation_file_list.txt"
154-
const baseURL = "http://store.carml.org.s3.amazonaws.com/datasets/ilsvrc2012_validation/"
155-
config.AfterInit(func() {
156-
157-
filePaths := strings.Split(_escFSMustString(false, fileListPath), "\n")
158-
159-
fileURLs := map[string]string{}
160-
for _, p := range filePaths {
161-
fileURLs[p] = baseURL + p
162-
}
163-
164-
iLSVRC2012Validation = &ILSVRC2012Validation{
165-
base: base{
166-
ctx: context.Background(),
167-
baseWorkingDir: filepath.Join(dldataset.Config.WorkingDirectory, "dldataset"),
168-
},
169-
baseURL: baseURL,
170-
fileURLs: fileURLs,
171-
filePaths: filePaths,
172-
}
173-
dldataset.Register(iLSVRC2012Validation)
174-
})
175-
}

‎vision/ilsvrc2012_validation_folder.go‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package vision
2+
3+
import (
4+
"net/http"
5+
"os"
6+
"path"
7+
"path/filepath"
8+
"strings"
9+
10+
"github.com/pkg/errors"
11+
"github.com/rai-project/config"
12+
"github.com/rai-project/dldataset"
13+
"github.com/rai-project/downloadmanager"
14+
"github.com/rai-project/image"
15+
"github.com/rai-project/image/types"
16+
context "golang.org/x/net/context"
17+
)
18+
19+
var iLSVRC2012Validation *ILSVRC2012Validation
20+
21+
// ILSVRC2012Validation ...
22+
type ILSVRC2012Validation struct {
23+
base
24+
baseURL string
25+
filePaths []string
26+
fileURLs map[string]string
27+
data map[string]ILSVRC2012ValidationLabeledImage
28+
}
29+
30+
// ILSVRC2012ValidationLabeledImage ...
31+
type ILSVRC2012ValidationLabeledImage struct {
32+
label string
33+
data *types.RGBImage
34+
}
35+
36+
// Label ...
37+
func (l ILSVRC2012ValidationLabeledImage) Label() string {
38+
return l.label
39+
}
40+
41+
// Data ...
42+
func (l ILSVRC2012ValidationLabeledImage) Data() (interface{}, error) {
43+
return l.data, nil
44+
}
45+
46+
// New ...
47+
func (d *ILSVRC2012Validation) New(ctx context.Context) (dldataset.Dataset, error) {
48+
return iLSVRC2012Validation, nil
49+
}
50+
51+
func (d *ILSVRC2012Validation) workingDir() string {
52+
category := strings.ToLower(d.Category())
53+
name := strings.ToLower(d.Name())
54+
return filepath.Join(d.baseWorkingDir, category, name)
55+
}
56+
57+
// Name ...
58+
func (d *ILSVRC2012Validation) Name() string {
59+
return "ilsvrc2012_validation_folder"
60+
}
61+
62+
// CanonicalName ...
63+
func (d *ILSVRC2012Validation) CanonicalName() string {
64+
category := strings.ToLower(d.Category())
65+
name := strings.ToLower(d.Name())
66+
key := path.Join(category, name)
67+
return key
68+
}
69+
70+
// Download ...
71+
func (d *ILSVRC2012Validation) Download(ctx context.Context) error {
72+
return nil
73+
}
74+
75+
// List ...
76+
func (d *ILSVRC2012Validation) List(ctx context.Context) ([]string, error) {
77+
return d.filePaths, nil
78+
}
79+
80+
// GetWithoutDownloadManager ...
81+
func (d *ILSVRC2012Validation) GetWithoutDownloadManager(ctx context.Context, name string) (dldataset.LabeledData, error) {
82+
fileURL, ok := d.fileURLs[name]
83+
if !ok {
84+
return nil, errors.Errorf("the file path %v for the dataset %v was not found", name, d.CanonicalName())
85+
}
86+
req, err := http.Get(fileURL)
87+
if err != nil {
88+
return nil, errors.Wrapf(err, "failed to perform http get request to %v", fileURL)
89+
}
90+
defer req.Body.Close()
91+
92+
img, err := image.Read(req.Body)
93+
if err != nil {
94+
return nil, errors.Wrapf(err, "failed to read image from %v", fileURL)
95+
}
96+
97+
if _, ok := img.(*types.RGBImage); !ok {
98+
return nil, errors.Wrapf(err, "failed to read rgb image from %v", fileURL)
99+
}
100+
101+
label := path.Dir(name)
102+
103+
return &ILSVRC2012ValidationLabeledImage{
104+
data: img.(*types.RGBImage),
105+
label: label,
106+
}, nil
107+
}
108+
109+
// Get ...
110+
func (d *ILSVRC2012Validation) Get(ctx context.Context, name string) (dldataset.LabeledData, error) {
111+
fileURL, ok := d.fileURLs[name]
112+
if !ok {
113+
return nil, errors.Errorf("the file path %v for the dataset %v was not found", name, d.CanonicalName())
114+
}
115+
116+
workingDir := d.workingDir()
117+
downloadedFileName := filepath.Join(workingDir, name)
118+
downloadedFileName, err := downloadmanager.DownloadFile(fileURL, downloadedFileName, downloadmanager.Context(ctx))
119+
if err != nil {
120+
return nil, errors.Wrapf(err, "failed to download %v", fileURL)
121+
}
122+
123+
f, err := os.Open(downloadedFileName)
124+
if err != nil {
125+
return nil, errors.Wrapf(err, "failed to open %v", downloadedFileName)
126+
}
127+
128+
defer f.Close()
129+
130+
img, err := image.Read(f)
131+
if err != nil {
132+
return nil, errors.Wrapf(err, "failed to read image from %v", fileURL)
133+
}
134+
135+
if _, ok := img.(*types.RGBImage); !ok {
136+
return nil, errors.Wrapf(err, "failed to read rgb image from %v", fileURL)
137+
}
138+
139+
label := path.Dir(name)
140+
141+
return &ILSVRC2012ValidationLabeledImage{
142+
data: img.(*types.RGBImage),
143+
label: label,
144+
}, nil
145+
}
146+
147+
// Close ...
148+
func (d *ILSVRC2012Validation) Close() error {
149+
return nil
150+
}
151+
152+
func init() {
153+
const fileListPath = "/vision/support/ilsvrc2012_validation_file_list.txt"
154+
const baseURL = "http://store.carml.org.s3.amazonaws.com/datasets/ilsvrc2012_validation/"
155+
config.AfterInit(func() {
156+
157+
filePaths := strings.Split(_escFSMustString(false, fileListPath), "\n")
158+
159+
fileURLs := map[string]string{}
160+
for _, p := range filePaths {
161+
fileURLs[p] = baseURL + p
162+
}
163+
164+
iLSVRC2012Validation = &ILSVRC2012Validation{
165+
base: base{
166+
ctx: context.Background(),
167+
baseWorkingDir: filepath.Join(dldataset.Config.WorkingDirectory, "dldataset"),
168+
},
169+
baseURL: baseURL,
170+
fileURLs: fileURLs,
171+
filePaths: filePaths,
172+
}
173+
dldataset.Register(iLSVRC2012Validation)
174+
})
175+
}

‎vision/ilsvrc2012_validation_test.go‎ renamed to ‎vision/ilsvrc2012_validation_folder_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ func TestILSVRC2012Validation(t *testing.T) {
1414

1515
ctx := context.Background()
1616

17-
ilsvrc, err := dldataset.Get("vision", "ilsvrc2012_validation")
17+
ilsvrc, err := dldataset.Get("vision", "ilsvrc2012_validation_folder")
1818
assert.NoError(t, err)
1919
assert.NotEmpty(t, ilsvrc)
2020

0 commit comments

Comments
 (0)