Skip to content

Commit eb5b062

Browse files
erizocosmicoerizocosmico
authored
Merge pull request #281 from mcarmonaa/feature/commit_blobs-table
gitbase: implement commit_blobs table
2 parents 2348e75 + 4727243 commit eb5b062

File tree

6 files changed

+391
-12
lines changed

6 files changed

+391
-12
lines changed

‎commit_blobs.go

+193
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
package gitbase
2+
3+
import (
4+
"io"
5+
6+
"gopkg.in/src-d/go-git.v4/plumbing/object"
7+
"gopkg.in/src-d/go-mysql-server.v0/sql"
8+
)
9+
10+
type commitBlobsTable struct{}
11+
12+
// CommitBlobsSchema is the schema for the commit blobs table.
13+
var CommitBlobsSchema = sql.Schema{
14+
{Name: "repository_id", Type: sql.Text, Source: CommitBlobsTableName},
15+
{Name: "commit_hash", Type: sql.Text, Source: CommitBlobsTableName},
16+
{Name: "blob_hash", Type: sql.Text, Source: CommitBlobsTableName},
17+
}
18+
19+
var _ sql.PushdownProjectionAndFiltersTable = (*commitBlobsTable)(nil)
20+
21+
func newCommitBlobsTable() sql.Table {
22+
return new(commitBlobsTable)
23+
}
24+
25+
func (commitBlobsTable) isGitbaseTable() {}
26+
27+
func (commitBlobsTable) String() string {
28+
return printTable(CommitBlobsTableName, CommitBlobsSchema)
29+
}
30+
31+
func (commitBlobsTable) Resolved() bool { return true }
32+
33+
func (commitBlobsTable) Name() string { return CommitBlobsTableName }
34+
35+
func (commitBlobsTable) Schema() sql.Schema { return CommitBlobsSchema }
36+
37+
func (t *commitBlobsTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) {
38+
return f(t)
39+
}
40+
41+
func (t *commitBlobsTable) TransformExpressionsUp(f sql.TransformExprFunc) (sql.Node, error) {
42+
return t, nil
43+
}
44+
45+
func (commitBlobsTable) Children() []sql.Node { return nil }
46+
47+
func (commitBlobsTable) RowIter(ctx *sql.Context) (sql.RowIter, error) {
48+
span, ctx := ctx.Span("gitbase.CommitBlobsTable")
49+
iter, err := NewRowRepoIter(ctx, &commitBlobsIter{})
50+
if err != nil {
51+
span.Finish()
52+
return nil, err
53+
}
54+
55+
return sql.NewSpanIter(span, iter), nil
56+
}
57+
58+
func (commitBlobsTable) HandledFilters(filters []sql.Expression) []sql.Expression {
59+
return handledFilters(CommitBlobsTableName, CommitBlobsSchema, filters)
60+
}
61+
62+
func (commitBlobsTable) WithProjectAndFilters(
63+
ctx *sql.Context,
64+
_, filters []sql.Expression,
65+
) (sql.RowIter, error) {
66+
span, ctx := ctx.Span("gitbase.CommitBlobsTable")
67+
iter, err := rowIterWithSelectors(
68+
ctx, CommitBlobsSchema, CommitBlobsTableName, filters,
69+
[]string{"commit_hash", "repository_id"},
70+
func(selectors selectors) (RowRepoIter, error) {
71+
repos, err := selectors.textValues("repository_id")
72+
if err != nil {
73+
return nil, err
74+
}
75+
76+
commits, err := selectors.textValues("commit_hash")
77+
if err != nil {
78+
return nil, err
79+
}
80+
81+
s, ok := ctx.Session.(*Session)
82+
if !ok {
83+
return nil, ErrInvalidGitbaseSession.New(ctx.Session)
84+
}
85+
86+
return &commitBlobsIter{
87+
repos: repos,
88+
commits: commits,
89+
skipGitErrors: s.SkipGitErrors,
90+
}, nil
91+
},
92+
)
93+
94+
if err != nil {
95+
span.Finish()
96+
return nil, err
97+
}
98+
99+
return sql.NewSpanIter(span, iter), nil
100+
}
101+
102+
type commitBlobsIter struct {
103+
repoID string
104+
iter object.CommitIter
105+
currCommit *object.Commit
106+
filesIter *object.FileIter
107+
skipGitErrors bool
108+
109+
// selectors for faster filtering
110+
repos []string
111+
commits []string
112+
}
113+
114+
func (i *commitBlobsIter) NewIterator(repo *Repository) (RowRepoIter, error) {
115+
var iter object.CommitIter
116+
if len(i.repos) == 0 || stringContains(i.repos, repo.ID) {
117+
var err error
118+
iter, err = NewCommitsByHashIter(repo, i.commits)
119+
if err != nil {
120+
return nil, err
121+
}
122+
}
123+
124+
return &commitBlobsIter{
125+
repoID: repo.ID,
126+
iter: iter,
127+
repos: i.repos,
128+
commits: i.commits,
129+
}, nil
130+
}
131+
132+
func (i *commitBlobsIter) Next() (sql.Row, error) {
133+
for {
134+
if i.iter == nil {
135+
return nil, io.EOF
136+
}
137+
138+
if i.currCommit == nil {
139+
commit, err := i.iter.Next()
140+
if err != nil {
141+
if err != io.EOF && i.skipGitErrors {
142+
continue
143+
}
144+
145+
return nil, err
146+
}
147+
148+
filesIter, err := commit.Files()
149+
if err != nil {
150+
if i.skipGitErrors {
151+
continue
152+
}
153+
154+
return nil, err
155+
}
156+
157+
i.currCommit = commit
158+
i.filesIter = filesIter
159+
}
160+
161+
file, err := i.filesIter.Next()
162+
if err != nil {
163+
if err == io.EOF {
164+
i.currCommit = nil
165+
i.filesIter.Close()
166+
i.filesIter = nil
167+
continue
168+
}
169+
170+
if i.skipGitErrors {
171+
continue
172+
}
173+
174+
return nil, err
175+
}
176+
177+
return sql.NewRow(
178+
i.repoID, i.currCommit.Hash.String(), file.Blob.Hash.String(),
179+
), nil
180+
}
181+
}
182+
183+
func (i *commitBlobsIter) Close() error {
184+
if i.filesIter != nil {
185+
i.filesIter.Close()
186+
}
187+
188+
if i.iter != nil {
189+
i.iter.Close()
190+
}
191+
192+
return nil
193+
}

‎commit_blobs_test.go

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
package gitbase
2+
3+
import (
4+
"io"
5+
"testing"
6+
7+
"github.com/stretchr/testify/require"
8+
"gopkg.in/src-d/go-mysql-server.v0/sql"
9+
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
10+
)
11+
12+
func TestCommitBlobsTableRowIter(t *testing.T) {
13+
require := require.New(t)
14+
15+
table := newCommitBlobsTable()
16+
require.IsType(&commitBlobsTable{}, table)
17+
18+
ctx, paths, cleanup := setupRepos(t)
19+
defer cleanup()
20+
21+
expectedRows := []sql.Row{
22+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
23+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
24+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
25+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "7e59600739c96546163833214c36459e324bad0a"),
26+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
27+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"),
28+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"),
29+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"),
30+
sql.NewRow(paths[0], "e8d3ffab552895c19b9fcf7aa264d277cde33881", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"),
31+
32+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
33+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
34+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
35+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
36+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"),
37+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"),
38+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"),
39+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"),
40+
sql.NewRow(paths[0], "6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "9dea2395f5403188298c1dabe8bdafe562c491e3"),
41+
42+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
43+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
44+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
45+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
46+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "880cd14280f4b9b6ed3986d6671f907d7cc2a198"),
47+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"),
48+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"),
49+
sql.NewRow(paths[0], "918c48b83bd081e863dbe1b80f8998f058cd8294", "9a48f23120e880dfbe41f7c9b7b708e9ee62a492"),
50+
51+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
52+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
53+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
54+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
55+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"),
56+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"),
57+
58+
sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
59+
sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
60+
sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
61+
sql.NewRow(paths[0], "1669dce138d9b841a518c64b10914d88f5e488ea", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
62+
63+
sql.NewRow(paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
64+
sql.NewRow(paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
65+
sql.NewRow(paths[0], "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
66+
67+
sql.NewRow(paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
68+
sql.NewRow(paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
69+
sql.NewRow(paths[0], "b8e471f58bcbca63b07bda20e428190409c2db47", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
70+
71+
sql.NewRow(paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
72+
sql.NewRow(paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
73+
sql.NewRow(paths[0], "35e85108805c84807bc66a02d91535e1e24b38b9", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
74+
75+
sql.NewRow(paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
76+
sql.NewRow(paths[0], "b029517f6300c2da0f4b651b8642506cd6aaf45d", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
77+
78+
sql.NewRow(paths[1], "47770b26e71b0f69c0ecd494b1066f8d1da4fc03", "278871477afb195f908155a65b5c651f1cfd02d3"),
79+
sql.NewRow(paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4", "278871477afb195f908155a65b5c651f1cfd02d3"),
80+
sql.NewRow(paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4", "b4f017e8c030d24aef161569b9ade3e55931ba01"),
81+
sql.NewRow(paths[1], "c7431b5bc9d45fb64a87d4a895ce3d1073c898d2", "97b013ecd2cc7f572960509f659d8068798d59ca"),
82+
sql.NewRow(paths[1], "f52d9c374365fec7f9962f11ebf517588b9e236e", "278871477afb195f908155a65b5c651f1cfd02d3"),
83+
}
84+
85+
rowIter, err := table.RowIter(ctx)
86+
require.NoError(err)
87+
defer func() { require.NoError(rowIter.Close()) }()
88+
89+
for _, expected := range expectedRows {
90+
row, err := rowIter.Next()
91+
require.NoError(err)
92+
require.Equal(expected, row)
93+
}
94+
95+
_, err = rowIter.Next()
96+
require.EqualError(err, io.EOF.Error())
97+
}
98+
99+
func TestCommitBlobsTablePushdown(t *testing.T) {
100+
require := require.New(t)
101+
102+
table := new(commitBlobsTable)
103+
require.NotNil(table)
104+
105+
ctx, paths, cleanup := setupRepos(t)
106+
defer cleanup()
107+
108+
var tests = []struct {
109+
name string
110+
filters []sql.Expression
111+
expectedRows []sql.Row
112+
}{
113+
{
114+
name: "commit_hash filter",
115+
filters: []sql.Expression{
116+
expression.NewEquals(
117+
expression.NewGetFieldWithTable(1, sql.Text, CommitBlobsTableName, "commit_hash", false),
118+
expression.NewLiteral("af2d6a6954d532f8ffb47615169c8fdf9d383a1a", sql.Text),
119+
),
120+
},
121+
expectedRows: []sql.Row{
122+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "32858aad3c383ed1ff0a0f9bdf231d54a00c9e88"),
123+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "d3ff53e0564a9f87d8e84b6e28e5060e517008aa"),
124+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "c192bd6a24ea1ab01d78686e417c8bdc7c3d197f"),
125+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "d5c0f4ab811897cadf03aec358ae60d21f91c50d"),
126+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "49c6bb89b17060d7b4deacb7b338fcc6ea2352a9"),
127+
sql.NewRow(paths[0], "af2d6a6954d532f8ffb47615169c8fdf9d383a1a", "c8f1d8c61f9da76f4cb49fd86322b6e685dba956"),
128+
},
129+
},
130+
{
131+
name: "repository_id filter",
132+
filters: []sql.Expression{
133+
expression.NewEquals(
134+
expression.NewGetFieldWithTable(1, sql.Text, CommitBlobsTableName, "repository_id", false),
135+
expression.NewLiteral(paths[1], sql.Text),
136+
),
137+
},
138+
expectedRows: []sql.Row{
139+
sql.NewRow(paths[1], "47770b26e71b0f69c0ecd494b1066f8d1da4fc03", "278871477afb195f908155a65b5c651f1cfd02d3"),
140+
sql.NewRow(paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4", "278871477afb195f908155a65b5c651f1cfd02d3"),
141+
sql.NewRow(paths[1], "b685400c1f9316f350965a5993d350bc746b0bf4", "b4f017e8c030d24aef161569b9ade3e55931ba01"),
142+
sql.NewRow(paths[1], "c7431b5bc9d45fb64a87d4a895ce3d1073c898d2", "97b013ecd2cc7f572960509f659d8068798d59ca"),
143+
sql.NewRow(paths[1], "f52d9c374365fec7f9962f11ebf517588b9e236e", "278871477afb195f908155a65b5c651f1cfd02d3"),
144+
},
145+
},
146+
}
147+
148+
for _, test := range tests {
149+
t.Run(test.name, func(t *testing.T) {
150+
rowIter, err := table.WithProjectAndFilters(ctx, nil, test.filters)
151+
require.NoError(err)
152+
defer func() { require.NoError(rowIter.Close()) }()
153+
154+
for _, expected := range test.expectedRows {
155+
row, err := rowIter.Next()
156+
require.NoError(err)
157+
require.Equal(expected, row)
158+
}
159+
160+
_, err = rowIter.Next()
161+
require.EqualError(err, io.EOF.Error())
162+
163+
})
164+
}
165+
}

‎commits.go

-4
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,6 @@ func (r *commitsTable) WithProjectAndFilters(
9090
ctx, CommitsSchema, CommitsTableName, filters,
9191
[]string{"commit_hash"},
9292
func(selectors selectors) (RowRepoIter, error) {
93-
if len(selectors["commit_hash"]) == 0 {
94-
return new(commitIter), nil
95-
}
96-
9793
hashes, err := selectors.textValues("commit_hash")
9894
if err != nil {
9995
return nil, err

0 commit comments

Comments
 (0)