Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: whywaita/myshoes
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v1.10.0
Choose a base ref
...
head repository: whywaita/myshoes
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
Loading
Showing with 4,034 additions and 1,987 deletions.
  1. +9 −8 .github/workflows/build-docker-sha.yaml
  2. +11 −18 .github/workflows/release.yaml
  3. +15 −12 .github/workflows/test.yaml
  4. +7 −7 Dockerfile
  5. +6 −4 Makefile
  6. +18 −6 README.md
  7. +38 −0 api/myshoes/README.md
  8. +82 −0 api/myshoes/client.go
  9. +54 −0 api/myshoes/http.go
  10. +106 −0 api/myshoes/target.go
  11. +119 −142 api/{proto → proto.go}/myshoes.pb.go
  12. +15 −6 api/{proto → proto.go}/myshoes_grpc.pb.go
  13. +8 −3 api/proto/myshoes.proto
  14. +22 −6 cmd/server/cmd.go
  15. +49 −4 docs/01_01_for_admin_setup.md
  16. +9 −0 docs/01_02_for_admin_tips.md
  17. +4 −19 docs/02_01_for_user_setup.md
  18. +71 −18 go.mod
  19. +199 −484 go.sum
  20. +0 −37 internal/config/config.go
  21. +0 −180 internal/config/init.go
  22. +1 −1 internal/testutils/testutils.go
  23. +19 −0 internal/util/util.go
  24. +122 −0 pkg/config/config.go
  25. +318 −0 pkg/config/init.go
  26. +141 −0 pkg/datastore/github.go
  27. +43 −80 pkg/datastore/interface.go
  28. +0 −48 pkg/datastore/interface_test.go
  29. +35 −21 pkg/datastore/memory/memory.go
  30. +9 −2 pkg/datastore/mysql/job.go
  31. +18 −9 pkg/datastore/mysql/job_test.go
  32. +1 −1 pkg/datastore/mysql/lock.go
  33. +5 −6 pkg/datastore/mysql/mysql.go
  34. +38 −4 pkg/datastore/mysql/runner.go
  35. +64 −9 pkg/datastore/mysql/runner_test.go
  36. +1 −4 pkg/datastore/mysql/schema.sql
  37. +8 −13 pkg/datastore/mysql/target.go
  38. +43 −145 pkg/datastore/mysql/target_test.go
  39. +1 −1 pkg/datastore/resource_type.go
  40. +112 −0 pkg/docker/ratelimit.go
  41. +106 −125 pkg/gh/github.go
  42. +44 −27 pkg/gh/github_test.go
  43. +118 −0 pkg/gh/installation.go
  44. +41 −99 pkg/gh/jwt.go
  45. +7 −15 pkg/gh/jwt_test.go
  46. +75 −0 pkg/gh/ratelimit.go
  47. +151 −0 pkg/gh/runner.go
  48. +78 −0 pkg/gh/token_registration.go
  49. +50 −0 pkg/gh/webhook.go
  50. +38 −0 pkg/gh/workflow_job.go
  51. +59 −0 pkg/gh/workflow_run.go
  52. +1 −1 pkg/logger/logger.go
  53. +1 −0 pkg/metric/collector.go
  54. +141 −6 pkg/metric/scrape_datastore.go
  55. +127 −0 pkg/metric/scrape_github.go
  56. +122 −5 pkg/metric/scrape_memory.go
  57. +50 −5 pkg/runner/runner.go
  58. +159 −44 pkg/runner/runner_delete.go
  59. +4 −3 pkg/runner/runner_delete_ephemeral.go
  60. +17 −5 pkg/runner/runner_delete_once.go
  61. +7 −6 pkg/runner/token_update.go
  62. +18 −9 pkg/shoes/shoes.go
  63. +50 −0 pkg/starter/error.go
  64. +29 −0 pkg/starter/metric.go
  65. +116 −35 pkg/starter/scripts.go
  66. +274 −46 pkg/starter/starter.go
  67. +1 −3 pkg/web/config.go
  68. +3 −4 pkg/web/http.go
  69. +2 −2 pkg/web/metrics.go
  70. +85 −110 pkg/web/target.go
  71. +36 −25 pkg/web/target_create.go
  72. +127 −63 pkg/web/target_test.go
  73. +76 −51 pkg/web/webhook.go
17 changes: 9 additions & 8 deletions .github/workflows/build-docker-sha.yaml
Original file line number Diff line number Diff line change
@@ -3,36 +3,37 @@ on:
push:
branches:
- "**"
workflow_dispatch:

jobs:
docker-build-sha:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- uses: docker/setup-qemu-action@v1
- uses: docker/setup-buildx-action@v1
uses: actions/checkout@v4
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v2
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/metadata-action@v3
- uses: docker/metadata-action@v5
id: meta
with:
images: ghcr.io/whywaita/myshoes
images: ghcr.io/${{ github.repository_owner }}/myshoes
tags: |
type=sha
- name: Build container image
uses: docker/build-push-action@v2
uses: docker/build-push-action@v5
with:
push: true
tags: ${{ steps.meta.outputs.tags }}
29 changes: 11 additions & 18 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
@@ -9,41 +9,34 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v2
uses: actions/setup-go@v5
with:
go-version: 1.16
go-version-file: 'go.mod'
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v2
uses: goreleaser/goreleaser-action@v6
with:
version: latest
args: release --rm-dist
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
docker:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- uses: docker/setup-qemu-action@v1
- uses: docker/setup-buildx-action@v1
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
uses: actions/checkout@v4
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/metadata-action@v3
- uses: docker/metadata-action@v5
id: meta
with:
images: ghcr.io/whywaita/myshoes
@@ -52,7 +45,7 @@ jobs:
type=semver,pattern={{raw}}
type=sha
- name: Build container image
uses: docker/build-push-action@v2
uses: docker/build-push-action@v5
with:
push: true
tags: ${{ steps.meta.outputs.tags }}
27 changes: 15 additions & 12 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
@@ -3,7 +3,8 @@ on:
push:
branches:
- "**"
pull_request: {}
pull_request:
workflow_dispatch:

jobs:
test:
@@ -13,20 +14,19 @@ jobs:
matrix:
os:
- ubuntu-latest
- ubuntu-18.04
steps:
- name: setup go
uses: actions/setup-go@v2
with:
go-version: 1.16.x
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: lint
run: |
GO111MODULE=off GOBIN=$(pwd)/bin go get golang.org/x/lint/golint
bin/golint -set_exit_status ./...
go install honnef.co/go/tools/cmd/staticcheck@latest
staticcheck ./...
- name: vet
run: |
go vet ./...
@@ -37,8 +37,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: docker build
run: docker build .
- name: Build container image
uses: docker/build-push-action@v5
with:
push: false
tags: ${{ steps.meta.outputs.tags }}
14 changes: 7 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
FROM golang:1.16 AS builder
FROM golang:1.23 AS builder

WORKDIR /go/src/github.com/whywaita/myshoes

RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1
RUN apt-get update -y \
&& apt-get install -y protobuf-compiler

ENV CGO_ENABLED=0
ENV GOOS=linux
ENV GOARCH=amd64

RUN go get -u google.golang.org/protobuf/cmd/protoc-gen-go \
google.golang.org/grpc/cmd/protoc-gen-go-grpc
RUN apt-get update -y \
&& apt-get install -y protobuf-compiler

COPY . .
RUN make build-linux

@@ -23,4 +23,4 @@ RUN apk add --no-cache ca-certificates \

COPY --from=builder /go/src/github.com/whywaita/myshoes/myshoes-linux-amd64 /app

CMD ["/app"]
CMD ["/app"]
10 changes: 6 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -18,10 +18,12 @@ build-linux: ## Build for Linux
GOOS=linux GOARCH=amd64 go build -o myshoes-linux-amd64 -ldflags $(BUILD_LDFLAGS) cmd/server/cmd.go

build-proto: ## Build proto file
protoc -I ./api/proto/ \
--go_out=./api/proto/ \
--go-grpc_out=./api/proto/ \
./api/proto/myshoes.proto
mkdir -p tmp/proto-go
rm -rf api/proto.go

protoc -I=api/proto/ --go_out=tmp/proto-go/ --go-grpc_out=tmp/proto-go/ api/proto/**.proto
mv tmp/proto-go/github.com/whywaita/myshoes/api/proto.go api/
rm -rf tmp

test: ## Exec test
go test -v ./...
24 changes: 18 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@

Auto scaling self-hosted runner :runner: (like GitHub-hosted) for GitHub Actions!

# features
## Features

- Auto-scaling and runner with your cloud-provider
- your infrastructure (private cloud, homelab...)
@@ -22,13 +22,25 @@ Auto scaling self-hosted runner :runner: (like GitHub-hosted) for GitHub Actions
- using special hardware
- Graphics Processing Unit (GPU)
- Field Programmable Gate Array (FPGA)
- And more in [whywaita/myshoes-providers](https://github.com/whywaita/myshoes-providers)

## Setup (only once)

Please check [setup.md](./docs/setup.md)
Please see [Documents](./docs).

## Known issue
## How to contribute

- sometimes occurred race condition if running multi jobs
- GitHub Action's runner has a problem like a race condition.
- related PullRequest: https://github.com/actions/runner/pull/660
1. Fork it
1. Clone original repository `git clone https://github.com/whywaita/myshoes`
1. Add remote your repository `git remote add your-name https://github.com/${your-name}/myshoes`
1. Create your feature branch `git switch -c my-new-feature`
1. Commit your changes `git commit -am 'Add some feature'`
1. Push to the branch `git push your-name my-new-feature`
1. Create new Pull Request

## Publications

### Talk

- [Development myshoes and Provide Cycloud-hosted runner -- GitHub Actions with your shoes. (en)](https://www.slideshare.net/whywaita/development-myshoes-and-provide-cycloudhosted-runner-github-actions-with-your-shoes)
- [Development OSS CI/CD platform in CyberAgent (ja)](https://www.slideshare.net/whywaita/cyberagent-oss-cicd-myshoes-cicd2021)
38 changes: 38 additions & 0 deletions api/myshoes/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# myshoes-sdk-go

The Go SDK for myshoes

## Usage

```go
package main

import (
"context"
"fmt"
"io"
"log"
"net/http"

"github.com/whywaita/myshoes/api/myshoes"
)

func main() {
// Set customized HTTP Client
customHTTPClient := http.DefaultClient
// Set customized logger
customLogger := log.New(io.Discard, "", log.LstdFlags)

client, err := myshoes.NewClient("https://example.com", customHTTPClient, customLogger)
if err != nil {
// ...
}

targets, err := client.ListTarget(context.Background())
if err != nil {
// ...
}

fmt.Println(targets)
}
```
82 changes: 82 additions & 0 deletions api/myshoes/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package myshoes

import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"strings"
)

// Client is a client for myshoes
type Client struct {
HTTPClient http.Client
URL *url.URL

UserAgent string
Logger *log.Logger
}

const (
defaultUserAgent = "myshoes-sdk-go"
)

// NewClient create a Client
func NewClient(endpoint string, client *http.Client, logger *log.Logger) (*Client, error) {
u, err := url.Parse(endpoint)
if err != nil {
return nil, fmt.Errorf("failed to parse endpoint: %w", err)
}

httpClient := client
if httpClient == nil {
httpClient = http.DefaultClient
}
l := logger
if l == nil {
return &Client{
HTTPClient: *httpClient,
URL: u,

// Default is discard logger
Logger: log.New(io.Discard, "", log.LstdFlags),
}, nil
}

return &Client{
HTTPClient: *httpClient,
URL: u,

Logger: l,
}, nil
}

func (c *Client) newRequest(ctx context.Context, method, spath string, body io.Reader) (*http.Request, error) {
u := *c.URL
u.Path = path.Join(c.URL.Path, spath)

req, err := http.NewRequestWithContext(ctx, method, u.String(), body)
if err != nil {
return nil, fmt.Errorf("failed to create a new HTTP request: %w", err)
}

ua := c.UserAgent
if strings.EqualFold(ua, "") {
ua = defaultUserAgent
}
req.Header.Set("User-Agent", ua)

req.Header.Set("Content-Type", "application/json")

return req, nil
}

// Error values
var (
errCreateRequest = "failed to create request: %w"
errRequest = "failed to request: %w"
errDecodeBody = "failed to decodeBody: %w"
)
Loading