diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..90ee3ec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" # Location of package manifests + schedule: + interval: "monthly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..e413706 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,21 @@ +name: Docker +on: + push: + tags: + - 'v*' +env: + IMAGE_NAME: ywserver/yiwen-api +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: Build the Docker image + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker buildx create --use + docker login --username ${{ secrets.CR_USERNAME }} --password ${{ secrets.CR_PASSWORD }} ${{ secrets.CR_REGISTRY }} + IMAGE_TAG="${{ secrets.CR_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}" + TAGS="-t ${IMAGE_TAG}" + docker buildx build --platform='linux/amd64,linux/arm64' $TAGS --push . diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..adb05bf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI +on: + push: + branches: + - 'main' +jobs: + # Test on various OS with default Go version. + tests: + name: Test on ${{matrix.os}} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + go-version: ['1.20.x'] + + steps: + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Print Go version + run: go version + + - name: Get dependencies + run: go get -v -t -d ./... + + - name: Run tests + run: go test -v -failfast -tags=test -timeout="3m" -race ./... diff --git a/.gitignore b/.gitignore index 3b735ec..08c42c9 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ # Go workspace file go.work +config.toml +debug +dist \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..99e8ef0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# Cross-compiling using Docker multi-platform builds/images and `xx`. +# +# https://docs.docker.com/build/building/multi-platform/ +# https://github.com/tonistiigi/xx +FROM --platform=${BUILDPLATFORM:-linux/amd64} tonistiigi/xx AS xx + +FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:bookworm AS builder +WORKDIR /src + +COPY --from=xx / / + +# `ARG`/`ENV` pair is a workaround for `docker build` backward-compatibility. +# +# https://github.com/docker/buildx/issues/510 +ARG BUILDPLATFORM +ENV BUILDPLATFORM=${BUILDPLATFORM:-linux/amd64} +RUN case "$BUILDPLATFORM" in \ + */amd64 ) PLATFORM=x86_64 ;; \ + */arm64 | */arm64/* ) PLATFORM=aarch64 ;; \ + * ) echo "Unexpected BUILDPLATFORM '$BUILDPLATFORM'" >&2; exit 1 ;; \ + esac; + +# `ARG`/`ENV` pair is a workaround for `docker build` backward-compatibility. +# +# https://github.com/docker/buildx/issues/510 +ARG TARGETPLATFORM +ENV TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64} + +COPY . . +RUN make xx-build + +FROM debian:bookworm-slim AS runtime + +RUN apt-get update \ + && apt-get install -y ca-certificates tzdata curl \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY --from=builder /src/config ./config +COPY --from=builder /src/dist/goapp ./ +ENV CONFIG_FILE_PATH=./config/config.toml + +ENTRYPOINT ["./goapp"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..58a04ca --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +# options +ignore_output = &> /dev/null + +.PHONY: run-dev test lint build xx-build + +APP_NAME := auth-api +APP_PATH := github.com/yiwen-ai/yiwen-api +APP_VERSION := $(shell git describe --tags --always --match "v[0-9]*") +BUILD_TIME := $(shell date -u +"%FT%TZ") +BUILD_COMMIT := $(shell git rev-parse HEAD) + +run-dev: + @CONFIG_FILE_PATH=${PWD}/config.toml APP_ENV=dev go run main.go + +test: + @CONFIG_FILE_PATH=${PWD}/config/test.yml APP_ENV=test go test ./... + +lint: + @hash golint > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + go get -u golang.org/x/lint/golint; \ + fi + @golint -set_exit_status ${PKG_LIST} + +build: + @mkdir -p ./dist + @CGO_ENABLED=0 go build -ldflags "-X ${APP_PATH}/src/conf.AppName=${APP_NAME} \ + -X ${APP_PATH}/src/conf.AppVersion=${APP_VERSION} \ + -X ${APP_PATH}/src/conf.BuildTime=${BUILD_TIME} \ + -X ${APP_PATH}/src/conf.GitSHA1=${BUILD_COMMIT}" \ + -o ./dist/goapp main.go + +xx-build: + @mkdir -p ./dist + @CGO_ENABLED=0 xx-go build -ldflags "-X ${APP_PATH}/src/conf.AppName=${APP_NAME} \ + -X ${APP_PATH}/src/conf.AppVersion=${APP_VERSION} \ + -X ${APP_PATH}/src/conf.BuildTime=${BUILD_TIME} \ + -X ${APP_PATH}/src/conf.GitSHA1=${BUILD_COMMIT}" \ + -o ./dist/goapp main.go + @xx-verify --static ./dist/goapp diff --git a/config/default.toml b/config/default.toml new file mode 100644 index 0000000..794a6fc --- /dev/null +++ b/config/default.toml @@ -0,0 +1,25 @@ +env = "test" # "test", "dev", "prod" + +[log] +# Log level: "trace", "debug", "info", "warn", "error" +level = "info" + +[server] +# The address to bind to. +addr = ":8080" +# The maximum number of seconds to wait for graceful shutdown. +graceful_shutdown = 10 + +[base] +userbase = "http://127.0.0.1:8080" +writing = "http://127.0.0.1:8080" +jarvis = "http://127.0.0.1:8080" +webscraper = "http://127.0.0.1:8080" + +[oss] +bucket = "yiwenai" +endpoint = "oss-cn-hangzhou.aliyuncs.com" +access_key_id = "" +access_key_secret = "" +prefix = "dev/cr/" +url_base = "https://cdn.yiwen.pub/" diff --git a/doc/api.md b/doc/api.md new file mode 100644 index 0000000..163f668 --- /dev/null +++ b/doc/api.md @@ -0,0 +1,51 @@ +# auth-api + +## 登录 + +### 发起登录 +`GET https://auth.yiwen.ai/idp/:idp/authorize?next_url=encodedUrl` + +其中 `idp` 为登录服务提供方，如 `github`, `google`, `wechat`。 + +`encodedUrl` 为可选参数，登录成功后会重定向到该地址，默认为 `https://www.yiwen.ai/login/state`。若提供，其域名必须为白名单中的域名。 + +例如通过 github Oauth2 登录： +``` +GET https://auth.yiwen.ai/idp/github/authorize +``` + +登录成功后，重定向 url 会追加 `status=200` 的参数，如重定向到： +``` +https://www.yiwen.ai/login/state?status=200 +``` + +登录失败时，重定向 url 会追加 `status` 和 `x-request-id` 参数，其中 `status` 为 4xx 或 5xx 的 http 状态码，`x-request-id` 可用于定位详细错误，如重定向到： +``` +https://www.yiwen.ai/login/state?status=403&x-request-id=xxxxxxx +``` + +### 登录成功获取用户信息 +`GET https://auth.yiwen.ai/userinfo` + +返回数据： +```json +{ + "cn": "clmby76knvc", // 用户名，全局唯一，但用户可以修改 + "name": "0xZensh", // 用户昵称 + "locale": "zho", // 用户默认语言 + "picture": "https://cdn.yiwen.pub/dev/pic/L3J7GFPpLLwapz0UF7H8wQ", + "status":0 // 用户状态，-2: Disabled -1: Suspended, 0: Normal, 1: Verified, 2: Protected +} +``` + +### 登录成功获取 access_token +`GET https://auth.yiwen.ai/access_token` + +返回数据： +```json +{ + "sub": "2f727b18-53e9-2cbc-1aa7-3d1417b1fcc1", // 用户在应用主体下的永久唯一标识，不同应用主体下 sub 不同 + "access_token": "hE2iAScESDIwM...YymcaaKQL8K", // access_token，用于 API 调用 + "expires_in": 3600 // 有效期 1 小时 +} +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..45cffb3 --- /dev/null +++ b/go.mod @@ -0,0 +1,39 @@ +module github.com/yiwen-ai/yiwen-api + +go 1.20 + +require ( + github.com/BurntSushi/toml v1.3.2 + github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible + github.com/fxamacker/cbor/v2 v2.5.0-beta5 + github.com/google/uuid v1.3.0 + github.com/klauspost/compress v1.16.7 + github.com/ldclabs/cose v1.1.1 + github.com/mssola/useragent v1.0.0 + github.com/rs/xid v1.5.0 + github.com/stretchr/testify v1.8.4 + github.com/teambition/compressible-go v1.0.1 + github.com/teambition/gear v1.27.2 + go.uber.org/dig v1.17.0 + golang.org/x/oauth2 v0.10.0 +) + +require ( + github.com/GitbookIO/mimedb v0.0.0-20180329142916-39fdfdb4def4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-http-utils/cookie v1.3.1 // indirect + github.com/go-http-utils/negotiator v1.0.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/teambition/trie-mux v1.5.2 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cda1d2c --- /dev/null +++ b/go.sum @@ -0,0 +1,101 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/GitbookIO/mimedb v0.0.0-20180329142916-39fdfdb4def4 h1:WM2ftu7PazeqAxMiT0j0XmYiWW1xX99v5/PbW2UKAzE= +github.com/GitbookIO/mimedb v0.0.0-20180329142916-39fdfdb4def4/go.mod h1:0JA2lIXs/dl3RUgHP5ivwjl3f0g+X2BQz3zWnq8IJa4= +github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= +github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= +github.com/fxamacker/cbor/v2 v2.5.0-beta5 h1:NldHpwv5bP+qnoI00fa/JAvGQ+68oEKxARR3PciaCdw= +github.com/fxamacker/cbor/v2 v2.5.0-beta5/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/go-http-utils/cookie v1.3.1 h1:GCdTeqVV5vDcjP7LrgYpH8pbt3dOYKS+Wrs7Jo3/k/w= +github.com/go-http-utils/cookie v1.3.1/go.mod h1:ATl4rfG3bEemjiVa+8WIfgNcBUWdYBTasfXKjJ3Avt8= +github.com/go-http-utils/negotiator v1.0.0 h1:Qp1zofD6Nw7KXApXa3pAjehP06Js0ILguEBCnHhZeVA= +github.com/go-http-utils/negotiator v1.0.0/go.mod h1:mTQe1sH0XhdFkeDiWpCY3QSk7Apo5jwOlIwLWJbJe2c= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/ldclabs/cose v1.1.1 h1:9vUZ272MebRkiysHN/7ovLPOD9KdOe9QkmCO9vtUXsE= +github.com/ldclabs/cose v1.1.1/go.mod h1:uMgRupgJLt3ckxx/G+U7XWJr1F0WxHfjlVeRlZa5u2M= +github.com/mssola/useragent v1.0.0 h1:WRlDpXyxHDNfvZaPEut5Biveq86Ze4o4EMffyMxmH5o= +github.com/mssola/useragent v1.0.0/go.mod h1:hz9Cqz4RXusgg1EdI4Al0INR62kP7aPSRNHnpU+b85Y= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/teambition/compressible-go v1.0.1 h1:9lLUAWKkLcuLGVLjVW0VzXacjH89px93PbkwmIw0lGU= +github.com/teambition/compressible-go v1.0.1/go.mod h1:K91wjCUqzpuY2ZpSi039mt4WzzjMGxPFMZHHTEoTvak= +github.com/teambition/gear v1.27.1 h1:2nNo/8IHujl87XSyVzgUwFPN7UJQbWdEhgqUQf+8yok= +github.com/teambition/gear v1.27.1/go.mod h1:H9UJHkhqeC/MCDFQ+PxpaSeMUz1p6pzBeLYzYMVsupM= +github.com/teambition/gear v1.27.2 h1:huxdTd3B/jnn04jjSuJm7+/lzwiulukN3Fd5ew1yn9w= +github.com/teambition/gear v1.27.2/go.mod h1:H9UJHkhqeC/MCDFQ+PxpaSeMUz1p6pzBeLYzYMVsupM= +github.com/teambition/trie-mux v1.5.2 h1:ALTagFwKZXkn1vfSRlODlmoZg+NMeWAm4dyBPQI6a8w= +github.com/teambition/trie-mux v1.5.2/go.mod h1:0Woh4KOHSN9bkJ66eWmLs8ltrEKw+fnZbFaHFfbMrtc= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..676f384 --- /dev/null +++ b/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/yiwen-ai/yiwen-api/src/api" + "github.com/yiwen-ai/yiwen-api/src/conf" + "github.com/yiwen-ai/yiwen-api/src/logging" +) + +var help = flag.Bool("help", false, "show help info") +var version = flag.Bool("version", false, "show version info") + +func main() { + flag.Parse() + if *help || *version { + data, _ := json.Marshal(api.GetVersion()) + fmt.Println(string(data)) + os.Exit(0) + } + + app := api.NewApp() + ctx := conf.Config.GlobalCtx + host := "http://" + conf.Config.Server.Addr + logging.Infof("%s@%s start on %s", conf.AppName, conf.AppVersion, host) + err := app.ListenWithContext(ctx, conf.Config.Server.Addr) + logging.Errf("%s@%s closed %v", conf.AppName, conf.AppVersion, err) +} diff --git a/src/api/app.go b/src/api/app.go new file mode 100644 index 0000000..96eb18d --- /dev/null +++ b/src/api/app.go @@ -0,0 +1,78 @@ +package api + +import ( + "log" + "strings" + "time" + + "github.com/fxamacker/cbor/v2" + "github.com/teambition/compressible-go" + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/conf" + "github.com/yiwen-ai/yiwen-api/src/logging" + "github.com/yiwen-ai/yiwen-api/src/util" +) + +// NewApp ... +func NewApp() *gear.App { + app := gear.New() + + app.Set(gear.SetTrustedProxy, true) + app.Set(gear.SetBodyParser, &bodyParser{gear.DefaultBodyParser(2 << 19)}) // 1mb + // ignore TLS handshake error + app.Set(gear.SetLogger, log.New(gear.DefaultFilterWriter(), "", 0)) + app.Set(gear.SetCompress, compressible.WithThreshold(256)) + app.Set(gear.SetGraceTimeout, time.Duration(conf.Config.Server.GracefulShutdown)*time.Second) + app.Set(gear.SetSender, &sendObject{}) + app.Set(gear.SetEnv, conf.Config.Env) + + app.UseHandler(logging.AccessLogger) + err := util.DigInvoke(func(routers []*gear.Router) error { + for _, router := range routers { + app.UseHandler(router) + } + return nil + }) + + if err != nil { + logging.Panicf("DigInvoke error: %v", err) + } + + return app +} + +type bodyParser struct { + inner gear.BodyParser +} + +func (d *bodyParser) MaxBytes() int64 { + return d.inner.MaxBytes() +} + +func (d *bodyParser) Parse(buf []byte, body any, mediaType, charset string) error { + if len(buf) == 0 { + return gear.ErrBadRequest.WithMsg("request entity empty") + } + + if strings.HasPrefix(mediaType, gear.MIMEApplicationCBOR) { + return cbor.Unmarshal(buf, body) + } + + return d.inner.Parse(buf, body, mediaType, charset) +} + +type sendObject struct{} + +func (s *sendObject) Send(ctx *gear.Context, code int, data any) error { + if strings.HasPrefix(ctx.GetHeader(gear.HeaderAccept), gear.MIMEApplicationCBOR) { + data, err := cbor.Marshal(data) + if err != nil { + return ctx.Error(err) + } + ctx.Type(gear.MIMEApplicationCBOR) + ctx.End(code, data) + } + + return ctx.JSON(code, data) +} diff --git a/src/api/creation.go b/src/api/creation.go new file mode 100644 index 0000000..77b70ac --- /dev/null +++ b/src/api/creation.go @@ -0,0 +1,40 @@ +package api + +import ( + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/bll" +) + +type Creation struct { + blls *bll.Blls +} + +func (a *Creation) Create(ctx *gear.Context) error { + // idp := ctx.Param("idp") + // xid := ctx.GetHeader(gear.HeaderXRequestID) + + // nextURL, ok := a.authURL.CheckNextUrl(ctx.Query("next_url")) + // if !ok { + // next := a.authURL.GenNextUrl(&nextURL, 400, xid) + // logging.SetTo(ctx, "error", fmt.Sprintf("invalid next_url %q", ctx.Query("next_url"))) + // return ctx.Redirect(next) + // } + + // provider, ok := a.providers[idp] + // if !ok { + // next := a.authURL.GenNextUrl(&nextURL, 400, xid) + // logging.SetTo(ctx, "error", fmt.Sprintf("unknown provider %q", idp)) + // return ctx.Redirect(next) + // } + + // state, err := a.createState(idp, provider.ClientID, nextURL.String()) + // if err != nil { + // next := a.authURL.GenNextUrl(&nextURL, 500, xid) + // logging.SetTo(ctx, "error", fmt.Sprintf("failed to create state: %v", err)) + // return ctx.Redirect(next) + // } + + // url := provider.AuthCodeURL(state) + return nil +} diff --git a/src/api/healthz.go b/src/api/healthz.go new file mode 100644 index 0000000..d2a683e --- /dev/null +++ b/src/api/healthz.go @@ -0,0 +1,34 @@ +package api + +import ( + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/bll" + "github.com/yiwen-ai/yiwen-api/src/conf" + "github.com/yiwen-ai/yiwen-api/src/logging" +) + +// Healthz .. +type Healthz struct { + blls *bll.Blls +} + +// Get .. +func (a *Healthz) Get(ctx *gear.Context) error { + stats, err := a.blls.Stats(ctx) + if err != nil { + return gear.ErrInternalServerError.From(err) + } + + logging.SetTo(ctx, "stats", stats) + return ctx.OkJSON(bll.SuccessResponse[map[string]string]{Result: GetVersion()}) +} + +func GetVersion() map[string]string { + return map[string]string{ + "name": conf.AppName, + "version": conf.AppVersion, + "buildTime": conf.BuildTime, + "gitSHA1": conf.GitSHA1, + } +} diff --git a/src/api/publication.go b/src/api/publication.go new file mode 100644 index 0000000..2f86c04 --- /dev/null +++ b/src/api/publication.go @@ -0,0 +1,15 @@ +package api + +import ( + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/bll" +) + +type Publication struct { + blls *bll.Blls +} + +func (a *Publication) Create(ctx *gear.Context) error { + return nil +} diff --git a/src/api/router.go b/src/api/router.go new file mode 100644 index 0000000..efa7a40 --- /dev/null +++ b/src/api/router.go @@ -0,0 +1,83 @@ +package api + +import ( + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/bll" + "github.com/yiwen-ai/yiwen-api/src/middleware" + "github.com/yiwen-ai/yiwen-api/src/util" +) + +func init() { + util.DigProvide(newAPIs) + util.DigProvide(newRouters) +} + +// APIs .. +type APIs struct { + Healthz *Healthz + Creation *Creation + Publication *Publication + Scraping *Scraping +} + +func newAPIs(blls *bll.Blls) *APIs { + return &APIs{ + Healthz: &Healthz{blls}, + Creation: &Creation{blls}, + Publication: &Publication{blls}, + Scraping: &Scraping{blls}, + } +} + +func todo(ctx *gear.Context) error { + return gear.ErrNotImplemented.WithMsgf("TODO: %s %s", ctx.Method, ctx.Path) +} + +func newRouters(apis *APIs) []*gear.Router { + + router := gear.NewRouter(gear.RouterOptions{ + Root: "/v1", + IgnoreCase: false, + FixedPathRedirect: false, + TrailingSlashRedirect: false, + }) + + router.Use(middleware.AuthToken(true).Auth) + + router.Get("/scraping", apis.Scraping.Create) + + router.Post("/creation", apis.Creation.Create) + router.Get("/creation", apis.Creation.Create) + router.Patch("/creation", apis.Creation.Create) + router.Delete("/creation", apis.Creation.Create) + + router.Post("/creation/list", apis.Creation.Create) + router.Patch("/creation/archive", apis.Creation.Create) + router.Patch("/creation/redraft", apis.Creation.Create) + router.Patch("/creation/review", todo) + router.Patch("/creation/approve", todo) + router.Patch("/creation/release", apis.Creation.Create) + router.Put("/creation/update_content", apis.Creation.Create) + router.Patch("/creation/update_content", todo) + router.Post("/creation/assist", todo) + + router.Post("/publication", apis.Publication.Create) + router.Get("/publication", apis.Publication.Create) + router.Patch("/publication", apis.Publication.Create) + router.Delete("/publication", apis.Publication.Create) + + router.Post("/publication/list", apis.Publication.Create) + router.Patch("/publication/archive", apis.Publication.Create) + router.Patch("/publication/redraft", apis.Publication.Create) + router.Patch("/publication/approve", todo) + router.Patch("/publication/publish", apis.Publication.Create) + router.Put("/publication/update_content", apis.Publication.Create) + router.Post("/publication/assist", todo) + + rx := gear.NewRouter() + // health check + rx.Get("/healthz", apis.Healthz.Get) + + return []*gear.Router{router, rx} +} diff --git a/src/api/scraping.go b/src/api/scraping.go new file mode 100644 index 0000000..604c110 --- /dev/null +++ b/src/api/scraping.go @@ -0,0 +1,20 @@ +package api + +import ( + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/bll" +) + +type Scraping struct { + blls *bll.Blls +} + +func (a *Scraping) Create(ctx *gear.Context) error { + targetUrl := ctx.Query("url") + output, err := a.blls.Webscraper.Create(ctx, targetUrl) + if err != nil { + return gear.ErrInternalServerError.From(err) + } + return ctx.OkSend(bll.SuccessResponse[bll.ScrapingOutput]{Result: *output}) +} diff --git a/src/bll/common.go b/src/bll/common.go new file mode 100644 index 0000000..22c55e6 --- /dev/null +++ b/src/bll/common.go @@ -0,0 +1,41 @@ +package bll + +import ( + "context" + + "github.com/yiwen-ai/yiwen-api/src/conf" + "github.com/yiwen-ai/yiwen-api/src/service" + "github.com/yiwen-ai/yiwen-api/src/util" +) + +func init() { + util.DigProvide(NewBlls) +} + +// Blls ... +type Blls struct { + Jarvis *Jarvis + Userbase *Userbase + Webscraper *Webscraper + Writing *Writing +} + +// NewBlls ... +func NewBlls(oss *service.OSS) *Blls { + cfg := conf.Config.Base + return &Blls{ + Jarvis: &Jarvis{svc: service.APIHost(cfg.Jarvis)}, + Userbase: &Userbase{svc: service.APIHost(cfg.Userbase), oss: oss}, + Webscraper: &Webscraper{svc: service.APIHost(cfg.Webscraper)}, + Writing: &Writing{svc: service.APIHost(cfg.Writing)}, + } +} + +func (b *Blls) Stats(ctx context.Context) (res map[string]any, err error) { + return b.Userbase.svc.Stats(ctx) +} + +type SuccessResponse[T any] struct { + Retry int `json:"retry" cbor:"retry"` + Result T `json:"result" cbor:"result"` +} diff --git a/src/bll/jarvis.go b/src/bll/jarvis.go new file mode 100644 index 0000000..2dd98b4 --- /dev/null +++ b/src/bll/jarvis.go @@ -0,0 +1,9 @@ +package bll + +import ( + "github.com/yiwen-ai/yiwen-api/src/service" +) + +type Jarvis struct { + svc service.APIHost +} diff --git a/src/bll/userbase.go b/src/bll/userbase.go new file mode 100644 index 0000000..6ca44aa --- /dev/null +++ b/src/bll/userbase.go @@ -0,0 +1,10 @@ +package bll + +import ( + "github.com/yiwen-ai/yiwen-api/src/service" +) + +type Userbase struct { + svc service.APIHost + oss *service.OSS +} diff --git a/src/bll/webscraper.go b/src/bll/webscraper.go new file mode 100644 index 0000000..80b305a --- /dev/null +++ b/src/bll/webscraper.go @@ -0,0 +1,49 @@ +package bll + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/yiwen-ai/yiwen-api/src/service" + "github.com/yiwen-ai/yiwen-api/src/util" +) + +type Webscraper struct { + svc service.APIHost +} + +type ScrapingOutput struct { + ID util.ID `json:"id" cbor:"id"` + Url string `json:"url" cbor:"url"` + Src string `json:"src" cbor:"src"` + Title string `json:"title" cbor:"title"` + Meta map[string]string `json:"meta" cbor:"meta"` + Content util.Raw `json:"content" cbor:"content"` +} + +func (b *Webscraper) Create(ctx context.Context, targetUrl string) (*ScrapingOutput, error) { + output := SuccessResponse[ScrapingOutput]{} + api := fmt.Sprintf("/v1/scraping?url=%s", url.QueryEscape(targetUrl)) + if err := b.svc.Get(ctx, api, &output); err != nil { + return nil, err + } + + time.Sleep(time.Duration(output.Retry) * time.Second) + api = fmt.Sprintf("/v1/document?id=%s&output=detail", output.Result.ID.String()) + i := 0 + for { + i += 1 + if err := b.svc.Get(ctx, api, &output); err != nil { + return nil, err + } + + if len(output.Result.Content) > 0 || i > 10 { + break + } + time.Sleep(time.Second) + } + + return &output.Result, nil +} diff --git a/src/bll/writing.go b/src/bll/writing.go new file mode 100644 index 0000000..a130ac0 --- /dev/null +++ b/src/bll/writing.go @@ -0,0 +1,9 @@ +package bll + +import ( + "github.com/yiwen-ai/yiwen-api/src/service" +) + +type Writing struct { + svc service.APIHost +} diff --git a/src/conf/config.go b/src/conf/config.go new file mode 100644 index 0000000..e48a9cb --- /dev/null +++ b/src/conf/config.go @@ -0,0 +1,108 @@ +package conf + +import ( + "context" + "fmt" + "math/rand" + "os" + "sync" + "time" + + "github.com/BurntSushi/toml" + "github.com/teambition/gear" +) + +// Config ... +var Config ConfigTpl + +var AppName = "yiwen-api" +var AppVersion = "0.1.0" +var BuildTime = "unknown" +var GitSHA1 = "unknown" + +var once sync.Once + +func init() { + p := &Config + readConfig(p) + if err := p.Validate(); err != nil { + panic(err) + } + p.Rand = rand.New(rand.NewSource(time.Now().UnixNano())) + p.GlobalCtx = gear.ContextWithSignal(context.Background()) +} + +type Logger struct { + Level string `json:"level" toml:"level"` +} + +type Server struct { + Addr string `json:"addr" toml:"addr"` + GracefulShutdown uint `json:"graceful_shutdown" toml:"graceful_shutdown"` +} + +type Base struct { + Userbase string `json:"userbase" toml:"userbase"` + Writing string `json:"writing" toml:"writing"` + Jarvis string `json:"jarvis" toml:"jarvis"` + Webscraper string `json:"webscraper" toml:"webscraper"` +} + +type OSS struct { + Bucket string `json:"bucket" toml:"bucket"` + Endpoint string `json:"endpoint" toml:"endpoint"` + AccessKeyId string `json:"access_key_id" toml:"access_key_id"` + AccessKeySecret string `json:"access_key_secret" toml:"access_key_secret"` + Prefix string `json:"prefix" toml:"prefix"` + UrlBase string `json:"url_base" toml:"url_base"` +} + +// ConfigTpl ... +type ConfigTpl struct { + Rand *rand.Rand + GlobalCtx context.Context + Env string `json:"env" toml:"env"` + Logger Logger `json:"log" toml:"log"` + Server Server `json:"server" toml:"server"` + Base Base `json:"base" toml:"base"` + OSS OSS `json:"oss" toml:"oss"` +} + +func (c *ConfigTpl) Validate() error { + return nil +} + +func readConfig(v interface{}, path ...string) { + once.Do(func() { + filePath, err := getConfigFilePath(path...) + if err != nil { + panic(err) + } + + data, err := os.ReadFile(filePath) + if err != nil { + panic(err) + } + + _, err = toml.Decode(string(data), v) + if err != nil { + panic(err) + } + }) +} + +func getConfigFilePath(path ...string) (string, error) { + // 优先使用的环境变量 + filePath := os.Getenv("CONFIG_FILE_PATH") + + // 或使用指定的路径 + if filePath == "" && len(path) > 0 { + filePath = path[0] + } + + if filePath == "" { + return "", fmt.Errorf("config file not specified") + } + + return filePath, nil +} diff --git a/src/logging/logger.go b/src/logging/logger.go new file mode 100644 index 0000000..16b7f11 --- /dev/null +++ b/src/logging/logger.go @@ -0,0 +1,69 @@ +package logging + +import ( + "fmt" + "os" + + "github.com/teambition/gear" + gearLogging "github.com/teambition/gear/logging" + "github.com/yiwen-ai/yiwen-api/src/conf" +) + +func init() { + Logger.SetJSONLog() + AccessLogger.SetJSONLog() + + // AccessLogger is not needed to set level. + err := gearLogging.SetLoggerLevel(Logger, conf.Config.Logger.Level) + if err != nil { + Logger.Err(err) + } +} + +// AccessLogger is used for access log +var AccessLogger = gearLogging.New(os.Stdout) + +// Logger is used for the server. +var Logger = gearLogging.New(os.Stderr) + +// SrvLog returns a Log with kind of server. +func SrvLog(format string, args ...interface{}) gearLogging.Log { + return gearLogging.Log{ + "kind": "server", + "message": fmt.Sprintf(format, args...), + } +} + +// Panicf produce a "Emergency" log into the Logger. +func Panicf(format string, args ...interface{}) { + Logger.Panic(SrvLog(format, args...)) +} + +// Errf produce a "Error" log into the Logger. +func Errf(format string, args ...interface{}) { + Logger.Err(SrvLog(format, args...)) +} + +// Warningf produce a "Warning" log into the Logger. +func Warningf(format string, args ...interface{}) { + Logger.Warning(SrvLog(format, args...)) +} + +// Infof produce a "Informational" log into the Logger. +func Infof(format string, args ...interface{}) { + Logger.Info(SrvLog(format, args...)) +} + +// Debugf produce a "Debug" log into the Logger. +func Debugf(format string, args ...interface{}) { + Logger.Debug(SrvLog(format, args...)) +} + +// FromCtx retrieve the Log instance for the AccessLogger. +func FromCtx(ctx *gear.Context) gearLogging.Log { + return AccessLogger.FromCtx(ctx) +} + +func SetTo(ctx *gear.Context, key string, val interface{}) { + AccessLogger.SetTo(ctx, key, val) +} diff --git a/src/middleware/auth.go b/src/middleware/auth.go new file mode 100644 index 0000000..705f388 --- /dev/null +++ b/src/middleware/auth.go @@ -0,0 +1,84 @@ +package middleware + +import ( + "net/http" + "strconv" + "strings" + + "github.com/teambition/gear" + "github.com/yiwen-ai/yiwen-api/src/util" +) + +type Session struct { + // cookie session 验证即可 + UserID util.ID + + // 以下字段需要 access token 验证 + AppID util.ID + UserStatus int + UserRating int + UserKind int + AppScope []string +} + +func (s *Session) HasToken() bool { + return s.AppID != util.ZeroID +} + +func (s *Session) HasScope(scope string) bool { + return util.StringSliceHas(s.AppScope, scope) +} + +type AuthToken bool + +func (m AuthToken) Auth(ctx *gear.Context) error { + sess, err := extractAuth(ctx) + if err != nil { + return gear.ErrUnauthorized.From(err) + } + + if bool(m) && !sess.HasToken() { + return gear.ErrUnauthorized.WithMsg("invalid token") + } + + ctxHeader := make(http.Header) + util.CopyHeader(ctxHeader, ctx.Req.Header, + "x-real-ip", + "x-request-id", + "x-auth-user", + "x-auth-user-rating", + "x-auth-app", + ) + + cctx := gear.CtxWith[Session](ctx.Context(), sess) + cheader := util.ContextHTTPHeader(ctxHeader) + ctx.WithContext(gear.CtxWith[util.ContextHTTPHeader](cctx, &cheader)) + return nil +} + +func extractAuth(ctx *gear.Context) (*Session, error) { + var err error + sess := &Session{} + sess.UserID, _ = util.ParseID(ctx.GetHeader("x-auth-user")) + if sess.UserID == util.ZeroID { + return nil, gear.ErrUnauthorized.WithMsg("invalid session") + } + + sess.AppID, err = util.ParseID(ctx.GetHeader("x-auth-app")) + if err == nil { + if sess.UserStatus, err = strconv.Atoi(ctx.GetHeader("x-auth-user-status")); err != nil { + return nil, gear.ErrUnauthorized.WithMsg("invalid user status") + } + if sess.UserRating, err = strconv.Atoi(ctx.GetHeader("x-auth-user-rating")); err != nil { + return nil, gear.ErrUnauthorized.WithMsg("invalid user rating") + } + if sess.UserKind, err = strconv.Atoi(ctx.GetHeader("x-auth-user-kind")); err != nil { + return nil, gear.ErrUnauthorized.WithMsg("invalid user kind") + } + if scope := ctx.GetHeader("x-auth-app-scope"); scope != "" { + sess.AppScope = strings.Split(scope, ",") + } + } + + return sess, nil +} diff --git a/src/service/base.go b/src/service/base.go new file mode 100644 index 0000000..ca9b951 --- /dev/null +++ b/src/service/base.go @@ -0,0 +1,36 @@ +package service + +import ( + "context" + "net/http" + + "github.com/yiwen-ai/yiwen-api/src/util" +) + +type APIHost string + +func (h APIHost) Stats(ctx context.Context) (map[string]any, error) { + res := make(map[string]any) + err := h.Get(ctx, "/healthz", &res) + return res, err +} + +func (h APIHost) Get(ctx context.Context, api string, output any) error { + return util.RequestCBOR(ctx, util.HTTPClient, http.MethodGet, string(h)+api, nil, output) +} + +func (h APIHost) Delete(ctx context.Context, api string, output any) error { + return util.RequestCBOR(ctx, util.HTTPClient, http.MethodDelete, string(h)+api, nil, output) +} + +func (h APIHost) Post(ctx context.Context, api string, input, output any) error { + return util.RequestCBOR(ctx, util.HTTPClient, http.MethodPost, string(h)+api, input, output) +} + +func (h APIHost) Put(ctx context.Context, api string, input, output any) error { + return util.RequestCBOR(ctx, util.HTTPClient, http.MethodPut, string(h)+api, input, output) +} + +func (h APIHost) Patch(ctx context.Context, api string, input, output any) error { + return util.RequestCBOR(ctx, util.HTTPClient, http.MethodPatch, string(h)+api, input, output) +} diff --git a/src/service/oss.go b/src/service/oss.go new file mode 100644 index 0000000..39fd080 --- /dev/null +++ b/src/service/oss.go @@ -0,0 +1,111 @@ +package service + +import ( + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/teambition/gear" + + "github.com/yiwen-ai/yiwen-api/src/conf" + "github.com/yiwen-ai/yiwen-api/src/util" +) + +func init() { + util.DigProvide(NewOSS) +} + +type OSS struct { + UrlBase string + Prefix string + bucket *oss.Bucket +} + +func NewOSS() *OSS { + cfg := conf.Config.OSS + client, err := oss.New(cfg.Endpoint, cfg.AccessKeyId, cfg.AccessKeySecret) + if err != nil { + panic(err) + } + bucket, err := client.Bucket(cfg.Bucket) + if err != nil { + panic(err) + } + + return &OSS{ + UrlBase: cfg.UrlBase, + Prefix: cfg.Prefix, + bucket: bucket, + } +} + +func (s *OSS) SavePicture(ctx context.Context, imgPath, imgUrl string) (string, error) { + ctype, reader, err := GetPicture(ctx, imgUrl) + if err != nil { + return "", err + } + + objectKey := s.Prefix + imgPath + if err := s.bucket.PutObject(objectKey, reader, oss.ContentType(ctype), + oss.CacheControl("public"), oss.ContentDisposition("inline")); err != nil { + return "", err + } + + return s.UrlBase + objectKey, nil +} + +func GetPicture(ctx context.Context, imgUrl string) (string, io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, "GET", imgUrl, nil) + if err != nil { + return "", nil, err + } + + resp, err := fileHTTPClient.Do(req) + if err != nil { + if err.(*url.Error).Unwrap() == context.Canceled { + return "", nil, gear.ErrClientClosedRequest + } + + return "", nil, err + } + + if resp.StatusCode != http.StatusOK { + data, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return "", nil, gear.Err.WithCode(resp.StatusCode).WithMsg(string(data)) + } + + ct := strings.ToLower(resp.Header.Get(gear.HeaderContentType)) + if !strings.Contains(ct, "image") { + resp.Body.Close() + return "", nil, gear.ErrUnsupportedMediaType.WithMsg(ct) + } + + return ct, resp.Body, nil +} + +var tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: false}, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 15 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 20, + IdleConnTimeout: 25 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 10 * time.Second, + ResponseHeaderTimeout: 15 * time.Second, +} + +var fileHTTPClient = &http.Client{ + Transport: tr, + Timeout: time.Second * 60, +} diff --git a/src/util/common.go b/src/util/common.go new file mode 100644 index 0000000..25054d1 --- /dev/null +++ b/src/util/common.go @@ -0,0 +1,11 @@ +package util + +// StringSliceHas ... +func StringSliceHas(sl []string, v string) bool { + for _, s := range sl { + if v == s { + return true + } + } + return false +} diff --git a/src/util/dig.go b/src/util/dig.go new file mode 100644 index 0000000..4c10b9d --- /dev/null +++ b/src/util/dig.go @@ -0,0 +1,16 @@ +package util + +// util 模块不要引入其它内部模块 +import "go.uber.org/dig" + +var globalDig = dig.New() + +// DigInvoke ... +func DigInvoke(function interface{}, opts ...dig.InvokeOption) error { + return globalDig.Invoke(function, opts...) +} + +// DigProvide ... +func DigProvide(constructor interface{}, opts ...dig.ProvideOption) error { + return globalDig.Provide(constructor, opts...) +} diff --git a/src/util/http.go b/src/util/http.go new file mode 100644 index 0000000..0caf870 --- /dev/null +++ b/src/util/http.go @@ -0,0 +1,190 @@ +package util + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "runtime" + "time" + + "github.com/fxamacker/cbor/v2" + "github.com/klauspost/compress/gzhttp" + "github.com/teambition/gear" +) + +func init() { + userAgent = fmt.Sprintf("Go/%v yiwen-api", runtime.Version()) +} + +type ContextHTTPHeader http.Header + +var userAgent string + +var externalTr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: false}, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 15 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 20, + IdleConnTimeout: 25 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 10 * time.Second, + ResponseHeaderTimeout: 15 * time.Second, +} + +var ExternalHTTPClient = &http.Client{ + Transport: gzhttp.Transport(externalTr), + Timeout: time.Second * 15, +} + +var internalTr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 15 * time.Second, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 25 * time.Second, + TLSHandshakeTimeout: 8 * time.Second, + ExpectContinueTimeout: 9 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, +} + +var HTTPClient = &http.Client{ + Transport: gzhttp.Transport(internalTr), + Timeout: time.Second * 5, +} + +var ErrNotFound = gear.ErrNotFound + +func RequestJSON(ctx context.Context, cli *http.Client, method, api string, input, output any) error { + if ctx.Err() != nil { + return nil + } + + var body io.Reader + if input != nil { + data, err := json.Marshal(input) + if err != nil { + return err + } + body = bytes.NewReader(data) + } + + req, err := http.NewRequestWithContext(ctx, method, api, body) + if err != nil { + return err + } + + req.Header.Set(gear.HeaderUserAgent, userAgent) + req.Header.Set(gear.HeaderAccept, gear.MIMEApplicationJSON) + if input != nil { + req.Header.Set(gear.HeaderContentType, gear.MIMEApplicationJSON) + } + + if header := gear.CtxValue[ContextHTTPHeader](ctx); header != nil { + CopyHeader(req.Header, http.Header(*header)) + } + + resp, err := cli.Do(req) + if err != nil { + if err.(*url.Error).Unwrap() == context.Canceled { + return gear.ErrClientClosedRequest + } + + return err + } + + defer resp.Body.Close() + if resp.StatusCode == 404 { + return ErrNotFound + } + + data, err := io.ReadAll(resp.Body) + if resp.StatusCode > 206 || err != nil { + return fmt.Errorf("RequestJSON %q failed, code: %d, error: %v, body: %s", + api, resp.StatusCode, err, string(data)) + } + + return json.Unmarshal(data, output) +} + +func RequestCBOR(ctx context.Context, cli *http.Client, method, api string, input, output any) error { + if ctx.Err() != nil { + return nil + } + + var body io.Reader + if input != nil { + data, err := cbor.Marshal(input) + if err != nil { + return err + } + body = bytes.NewReader(data) + } + + req, err := http.NewRequestWithContext(ctx, method, api, body) + if err != nil { + return err + } + + req.Header.Set(gear.HeaderUserAgent, userAgent) + req.Header.Set(gear.HeaderAccept, gear.MIMEApplicationCBOR) + if input != nil { + req.Header.Set(gear.HeaderContentType, gear.MIMEApplicationCBOR) + } + + if header := gear.CtxValue[ContextHTTPHeader](ctx); header != nil { + CopyHeader(req.Header, http.Header(*header)) + } + + resp, err := cli.Do(req) + if err != nil { + if err.(*url.Error).Unwrap() == context.Canceled { + return gear.ErrClientClosedRequest + } + + return err + } + + defer resp.Body.Close() + if resp.StatusCode == 404 { + return ErrNotFound + } + + data, err := io.ReadAll(resp.Body) + if resp.StatusCode > 206 || err != nil { + return fmt.Errorf("RequestCBOR %q failed, code: %d, error: %v, body: %s", + api, resp.StatusCode, err, string(data)) + } + + return cbor.Unmarshal(data, output) +} + +func CopyHeader(dst http.Header, src http.Header, names ...string) { + for k, vv := range src { + if len(names) > 0 && !StringSliceHas(names, k) { + continue + } + + switch len(vv) { + case 1: + dst.Set(k, vv[0]) + default: + dst.Del(k) + for _, v := range vv { + dst.Add(k, v) + } + } + } +} diff --git a/src/util/id.go b/src/util/id.go new file mode 100644 index 0000000..ef9f620 --- /dev/null +++ b/src/util/id.go @@ -0,0 +1,154 @@ +package util + +// util 模块不要引入其它内部模块 +import ( + "encoding/base64" + "errors" + "strconv" + + "github.com/fxamacker/cbor/v2" + "github.com/google/uuid" + "github.com/rs/xid" +) + +var ZeroID ID +var JARVIS ID = mustParseID("0000000000000jarvis0") // system user +var ANON ID = mustParseID("000000000000000anon0") // anonymous user + +func ParseID(s string) (ID, error) { + id, err := xid.FromString(s) + if err != nil { + return ZeroID, err + } + return ID(id), nil +} + +func mustParseID(s string) ID { + id, err := xid.FromString(s) + if err != nil { + panic(err) + } + return ID(id) +} + +type ID xid.ID + +func (id ID) String() string { + return xid.ID(id).String() +} + +func (id ID) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(xid.ID(id).Bytes()) +} + +func (id *ID) UnmarshalCBOR(data []byte) error { + if id == nil { + return errors.New("util.ID.UnmarshalCBOR: nil pointer") + } + + var buf []byte + if err := cbor.Unmarshal(data, &buf); err != nil { + return errors.New("util.ID.UnmarshalCBOR: " + err.Error()) + } + + if bytesLen := len(buf); bytesLen != 12 { + return errors.New("util.ID.UnmarshalCBOR: invalid bytes length, expected " + + strconv.Itoa(12) + ", got " + strconv.Itoa(bytesLen)) + } + + copy((*id)[:], buf) + return nil +} + +func (id ID) MarshalJSON() ([]byte, error) { + return xid.ID(id).MarshalJSON() +} + +func (id *ID) UnmarshalJSON(data []byte) error { + return (*xid.ID)(id).UnmarshalJSON(data) +} + +type UUID uuid.UUID + +func (id UUID) String() string { + return uuid.UUID(id).String() +} + +func (id UUID) Base64() string { + return base64.RawURLEncoding.EncodeToString(id[:]) +} + +func (id UUID) MarshalCBOR() ([]byte, error) { + data, _ := uuid.UUID(id).MarshalBinary() + return cbor.Marshal(data) +} + +func (id *UUID) UnmarshalCBOR(data []byte) error { + if id == nil { + return errors.New("util.UUID.UnmarshalCBOR: nil pointer") + } + + var buf []byte + if err := cbor.Unmarshal(data, &buf); err != nil { + return errors.New("util.UUID.UnmarshalCBOR: " + err.Error()) + } + + if bytesLen := len(buf); bytesLen != 16 { + return errors.New("util.UUID.UnmarshalCBOR: invalid bytes length, expected " + + strconv.Itoa(12) + ", got " + strconv.Itoa(bytesLen)) + } + + copy((*id)[:], buf) + return nil +} + +func (id UUID) MarshalText() ([]byte, error) { + return uuid.UUID(id).MarshalText() +} + +func (id *UUID) UnmarshalText(data []byte) error { + return (*uuid.UUID)(id).UnmarshalText(data) +} + +type Raw []byte + +func (r Raw) String() string { + return base64.RawURLEncoding.EncodeToString(r) +} + +func (r Raw) MarshalCBOR() ([]byte, error) { + if len(r) == 0 { + return []byte{0xf6}, nil + } + return r, nil +} + +func (r *Raw) UnmarshalCBOR(data []byte) error { + if r == nil { + return errors.New("util.Raw: UnmarshalCBOR on nil pointer") + } + *r = append((*r)[0:0], data...) + return nil +} + +func (r Raw) MarshalJSON() ([]byte, error) { + if len(r) == 0 { + return []byte("null"), nil + } + + return []byte("\"" + base64.RawURLEncoding.EncodeToString(r) + "\""), nil +} + +func (r *Raw) UnmarshalJSON(data []byte) error { + if r == nil { + return errors.New("util.Raw: UnmarshalJSON on nil pointer") + } + if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { + return errors.New("util.Raw: UnmarshalJSON with invalid data") + } + data, err := base64.RawURLEncoding.DecodeString(string(data[1 : len(data)-1])) + if err == nil { + *r = append((*r)[0:0], data...) + } + return err +} diff --git a/src/util/id_test.go b/src/util/id_test.go new file mode 100644 index 0000000..bb2d6db --- /dev/null +++ b/src/util/id_test.go @@ -0,0 +1,61 @@ +// (c) 2022-present, Yiwen AI, LLC. All rights reserved. +// See the file LICENSE for licensing terms. + +package util + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/fxamacker/cbor/v2" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func TestID(t *testing.T) { + t.Run("CBOR", func(t *testing.T) { + assert := assert.New(t) + + data, err := cbor.Marshal(JARVIS) + assert.NoError(err) + var id ID + assert.NoError(cbor.Unmarshal(data, &id)) + assert.Equal(JARVIS, id) + }) + + t.Run("JSON", func(t *testing.T) { + assert := assert.New(t) + + data, err := json.Marshal(ANON) + assert.NoError(err) + assert.Equal(`"000000000000000anon0"`, string(data)) + var id ID + assert.NoError(json.Unmarshal(data, &id)) + assert.Equal(ANON, id) + }) +} + +func TestUUID(t *testing.T) { + uid := UUID(uuid.Must(uuid.NewUUID())) + t.Run("CBOR", func(t *testing.T) { + assert := assert.New(t) + + data, err := cbor.Marshal(uid) + assert.NoError(err) + var id UUID + assert.NoError(cbor.Unmarshal(data, &id)) + assert.Equal(uid, id) + }) + + t.Run("JSON", func(t *testing.T) { + assert := assert.New(t) + + data, err := json.Marshal(uid) + assert.NoError(err) + assert.Equal(strconv.Quote(uid.String()), string(data)) + var id UUID + assert.NoError(json.Unmarshal(data, &id)) + assert.Equal(uid, id) + }) +}