Skip to content
This repository was archived by the owner on Feb 4, 2021. It is now read-only.

Commit c4042c1

Browse files
gedorinkugedorinku
committed
Impl github job
1 parent 788b330 commit c4042c1

File tree

11 files changed

+287
-5
lines changed

11 files changed

+287
-5
lines changed

‎.env.ci

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,5 @@ CLIENT_REGISTRATION_URL=http://localhost:8080/registration
3535

3636
SMTP_ADDR=smtp:1025
3737
EMAIL_FROM=[email protected]
38+
39+
GITHUB_ACCESS_TOKEN=yourAccessToken

‎.env.sample

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,5 @@ CLIENT_REGISTRATION_URL=http://localhost:8080/registration
3636

3737
SMTP_ADDR=smtp:1025
3838
EMAIL_FROM=[email protected]
39+
40+
GITHUB_ACCESS_TOKEN=yourAccessToken

‎app/config/config.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ type Config struct {
2222
ClientRegistrationURL string `envconfig:"client_registration_url" required:"true"`
2323
SMTPAddr string `envconfig:"smtp_addr" required:"true"`
2424
EmailFrom string `envconfig:"email_from" required:"true"`
25+
GitHubAccessToken string `envconfig:"github_access_token" required:"true"`
2526
}
2627

2728
// LoadConfig loads config

‎app/di/store_component.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
departmentstore "github.com/ProgrammingLab/prolab-accounts/infra/store/department"
1515
entrystore "github.com/ProgrammingLab/prolab-accounts/infra/store/entry"
1616
feedstore "github.com/ProgrammingLab/prolab-accounts/infra/store/feed"
17+
githubstore "github.com/ProgrammingLab/prolab-accounts/infra/store/github"
1718
heartbeatstore "github.com/ProgrammingLab/prolab-accounts/infra/store/heartbeat"
1819
invitationstore "github.com/ProgrammingLab/prolab-accounts/infra/store/invitation"
1920
profilestore "github.com/ProgrammingLab/prolab-accounts/infra/store/profile"
@@ -35,6 +36,7 @@ type StoreComponent interface {
3536
HeartbeatStore(ctx context.Context) store.HeartbeatStore
3637
DepartmentStore(ctx context.Context) store.DepartmentStore
3738
InvitationStore(ctx context.Context) store.InvitationStore
39+
GitHubStore(ctx context.Context) store.GitHubStore
3840
}
3941

4042
// NewStoreComponent returns new store component
@@ -175,3 +177,7 @@ func (s *storeComponentImpl) HeartbeatStore(ctx context.Context) store.Heartbeat
175177
func (s *storeComponentImpl) InvitationStore(ctx context.Context) store.InvitationStore {
176178
return invitationstore.NewInvitationStore(ctx, s.db)
177179
}
180+
181+
func (s *storeComponentImpl) GitHubStore(ctx context.Context) store.GitHubStore {
182+
return githubstore.NewGitHubStore(ctx, s.db, s.client)
183+
}

‎app/job/feed_job.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ import (
66

77
"google.golang.org/grpc/grpclog"
88

9+
"github.com/ProgrammingLab/prolab-accounts/app/config"
910
"github.com/ProgrammingLab/prolab-accounts/app/di"
1011
)
1112

12-
func feedJob(ctx context.Context, store di.StoreComponent, debug bool) error {
13+
func feedJob(ctx context.Context, store di.StoreComponent, cfg *config.Config) error {
1314
bs := store.UserBlogStore(ctx)
1415
blogs, err := bs.ListUserBlogs()
1516
if err != nil {
@@ -32,7 +33,7 @@ func feedJob(ctx context.Context, store di.StoreComponent, debug bool) error {
3233
if err != nil {
3334
return err
3435
}
35-
if debug {
36+
if cfg.DebugLog {
3637
grpclog.Infof("feed job: created %v entries", n)
3738
}
3839

‎app/job/github_job.go

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package job
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"github.com/pkg/errors"
8+
"github.com/shurcooL/githubv4"
9+
"golang.org/x/oauth2"
10+
11+
"github.com/ProgrammingLab/prolab-accounts/app/config"
12+
"github.com/ProgrammingLab/prolab-accounts/app/di"
13+
"github.com/ProgrammingLab/prolab-accounts/infra/record"
14+
"github.com/ProgrammingLab/prolab-accounts/model"
15+
)
16+
17+
type githubUser struct {
18+
User struct {
19+
ContributionsCollection struct {
20+
ContributionCalendar struct {
21+
TotalContributions githubv4.Int
22+
Weeks []struct {
23+
ContributionDays []struct {
24+
ContributionCount githubv4.Int
25+
Date githubv4.String
26+
}
27+
}
28+
}
29+
} `graphql:"contributionsCollection(from:$from, to:$to)"`
30+
} `graphql:"user(login:$login)"`
31+
}
32+
33+
const (
34+
contributionsFromDay = 60
35+
githubDateFormat = "2006-1-2"
36+
)
37+
38+
func githubJob(ctx context.Context, store di.StoreComponent, cfg *config.Config) error {
39+
src := oauth2.StaticTokenSource(
40+
&oauth2.Token{AccessToken: cfg.GitHubAccessToken},
41+
)
42+
httpClient := oauth2.NewClient(ctx, src)
43+
cli := githubv4.NewClient(httpClient)
44+
45+
us := store.UserStore(ctx)
46+
next := model.UserID(1)
47+
to := time.Now().UTC()
48+
from := to.AddDate(0, 0, -contributionsFromDay).Round(time.Hour * 24)
49+
for next != 0 {
50+
users, nxt, err := us.ListPublicUsers(next, 100)
51+
next = nxt
52+
if err != nil {
53+
return err
54+
}
55+
56+
for _, u := range users {
57+
name := u.R.Profile.GithubUserName
58+
if name.IsZero() {
59+
continue
60+
}
61+
gu, err := getGitHubUser(ctx, cli, name.String, from, to)
62+
if err != nil {
63+
return err
64+
}
65+
66+
err = storeGitHubContributions(ctx, store, u, gu)
67+
if err != nil {
68+
return err
69+
}
70+
}
71+
}
72+
73+
return nil
74+
}
75+
76+
func getGitHubUser(ctx context.Context, cli *githubv4.Client, name string, from time.Time, to time.Time) (*githubUser, error) {
77+
v := map[string]interface{}{
78+
"login": githubv4.String(name),
79+
"from": githubv4.DateTime{Time: from},
80+
"to": githubv4.DateTime{Time: to},
81+
}
82+
q := &githubUser{}
83+
err := cli.Query(ctx, q, v)
84+
if err != nil {
85+
return nil, errors.WithStack(err)
86+
}
87+
88+
return q, nil
89+
}
90+
91+
func storeGitHubContributions(ctx context.Context, store di.StoreComponent, user *record.User, github *githubUser) error {
92+
days := make([]*record.GithubContributionDay, 0, contributionsFromDay)
93+
weeks := github.User.ContributionsCollection.ContributionCalendar.Weeks
94+
for _, w := range weeks {
95+
for _, d := range w.ContributionDays {
96+
date, err := time.Parse(githubDateFormat, string(d.Date))
97+
if err != nil {
98+
return errors.WithStack(err)
99+
}
100+
gd := &record.GithubContributionDay{
101+
Count: int(d.ContributionCount),
102+
Date: date,
103+
UserID: user.ID,
104+
}
105+
days = append(days, gd)
106+
}
107+
}
108+
109+
c := &model.GitHubContributionCollection{
110+
UserID: model.UserID(user.ID),
111+
TotalCount: int(github.User.ContributionsCollection.ContributionCalendar.TotalContributions),
112+
Days: days,
113+
}
114+
gs := store.GitHubStore(ctx)
115+
_, err := gs.UpdateContributionDays(c)
116+
return err
117+
}

‎app/job/heartbeat_job.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@ import (
55

66
"github.com/pkg/errors"
77

8+
"github.com/ProgrammingLab/prolab-accounts/app/config"
89
"github.com/ProgrammingLab/prolab-accounts/app/di"
910
)
1011

11-
func heartbeatJob(ctx context.Context, store di.StoreComponent, debug bool) error {
12+
func heartbeatJob(ctx context.Context, store di.StoreComponent, cfg *config.Config) error {
1213
s := store.HeartbeatStore(ctx)
1314
return errors.WithStack(s.Beat())
1415
}

‎app/job/job.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@ var (
1616
stop = make(chan struct{})
1717
jobs = []Job{
1818
feedJob,
19+
githubJob,
1920
heartbeatJob,
2021
}
2122
)
2223

2324
// Job represents job for worker
24-
type Job func(ctx context.Context, store di.StoreComponent, debug bool) error
25+
type Job func(ctx context.Context, store di.StoreComponent, cfg *config.Config) error
2526

2627
// Start starts the worker
2728
func Start(store di.StoreComponent, cfg *config.Config) {
@@ -58,7 +59,7 @@ func run(store di.StoreComponent, cfg *config.Config) {
5859
select {
5960
case <-time.After(interval):
6061
for _, j := range jobs {
61-
err := j(context.Background(), store, cfg.DebugLog)
62+
err := j(context.Background(), store, cfg)
6263
if err != nil {
6364
grpclog.Errorf("job error: %+v", err)
6465
}

‎infra/store/github/github_store.go

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package githubstore
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"fmt"
7+
"strings"
8+
"time"
9+
10+
"github.com/go-redis/redis"
11+
"github.com/pkg/errors"
12+
"github.com/volatiletech/sqlboiler/boil"
13+
14+
"github.com/ProgrammingLab/prolab-accounts/infra/record"
15+
"github.com/ProgrammingLab/prolab-accounts/infra/store"
16+
"github.com/ProgrammingLab/prolab-accounts/model"
17+
"github.com/ProgrammingLab/prolab-accounts/sqlutil"
18+
)
19+
20+
type githubStoreImpl struct {
21+
ctx context.Context
22+
db *sqlutil.DB
23+
cli *redis.Client
24+
}
25+
26+
// NewGitHubStore returns new github store
27+
func NewGitHubStore(ctx context.Context, db *sql.DB, cli *redis.Client) store.GitHubStore {
28+
return &githubStoreImpl{
29+
ctx: ctx,
30+
db: sqlutil.New(db),
31+
cli: cli,
32+
}
33+
}
34+
35+
const (
36+
contributionTotalCount = "contributions-total-count"
37+
)
38+
39+
func (s *githubStoreImpl) UpdateContributionDays(c *model.GitHubContributionCollection) ([]*record.GithubContributionDay, error) {
40+
z := redis.Z{
41+
Score: float64(c.TotalCount),
42+
Member: int64(c.UserID),
43+
}
44+
err := s.cli.ZAdd(contributionTotalCount, z).Err()
45+
if err != nil {
46+
return nil, errors.WithStack(err)
47+
}
48+
49+
days := c.Days
50+
err = s.db.Watch(s.ctx, func(ctx context.Context, tx *sql.Tx) error {
51+
q := record.GithubContributionDayWhere.UserID.EQ(int64(c.UserID))
52+
_, err := record.GithubContributionDays(q).DeleteAll(ctx, tx)
53+
if err != nil {
54+
return errors.WithStack(err)
55+
}
56+
57+
return bulkInsert(ctx, tx, days)
58+
})
59+
60+
if err != nil {
61+
return nil, err
62+
}
63+
64+
return days, nil
65+
}
66+
67+
func bulkInsert(ctx context.Context, tx *sql.Tx, days []*record.GithubContributionDay) error {
68+
if len(days) == 0 {
69+
return nil
70+
}
71+
72+
q := &strings.Builder{}
73+
_, err := q.WriteString("INSERT INTO " + record.TableNames.GithubContributionDays +
74+
" (count, date, user_id, created_at, updated_at) VALUES ")
75+
if err != nil {
76+
return errors.WithStack(err)
77+
}
78+
79+
verbs, v := insertValueQuery(1)
80+
_, err = q.WriteString(v)
81+
if err != nil {
82+
return errors.WithStack(err)
83+
}
84+
for i := 0; i < len(days)-1; i++ {
85+
nxt, v := insertValueQuery(verbs)
86+
verbs = nxt
87+
_, err := q.WriteString(", " + v)
88+
if err != nil {
89+
return errors.WithStack(err)
90+
}
91+
}
92+
_, err = q.WriteString(";")
93+
if err != nil {
94+
return errors.WithStack(err)
95+
}
96+
97+
stmt, err := tx.Prepare(q.String())
98+
if err != nil {
99+
return errors.WithStack(err)
100+
}
101+
102+
values := make([]interface{}, 0, len(days)*5)
103+
now := time.Now().In(boil.GetLocation())
104+
for _, d := range days {
105+
values = append(values, insertValues(d, now, now)...)
106+
}
107+
108+
if boil.DebugMode {
109+
fmt.Fprintln(boil.DebugWriter, q.String())
110+
fmt.Fprintln(boil.DebugWriter, values)
111+
}
112+
_, err = stmt.ExecContext(ctx, values...)
113+
return errors.WithStack(err)
114+
}
115+
116+
func insertValueQuery(verbs int) (int, string) {
117+
return verbs + 5, fmt.Sprintf("($%v, $%v, $%v, $%v, $%v)", verbs, verbs+1, verbs+2, verbs+3, verbs+4)
118+
}
119+
120+
func insertValues(d *record.GithubContributionDay, createdAt, updatedAt time.Time) []interface{} {
121+
return []interface{}{
122+
d.Count,
123+
d.Date,
124+
d.UserID,
125+
createdAt,
126+
updatedAt,
127+
}
128+
}

‎infra/store/github_store.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package store
2+
3+
import (
4+
"github.com/ProgrammingLab/prolab-accounts/infra/record"
5+
"github.com/ProgrammingLab/prolab-accounts/model"
6+
)
7+
8+
// GitHubStore provides github data
9+
type GitHubStore interface {
10+
UpdateContributionDays(c *model.GitHubContributionCollection) ([]*record.GithubContributionDay, error)
11+
}

0 commit comments

Comments
 (0)