-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #326 from CSUSTers/dev
feat(inline): remove tracing parameters from Bilibili URL
- Loading branch information
Showing
20 changed files
with
2,892 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| package inline | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "csust-got/util/urlx" | ||
| "net/http" | ||
| "net/url" | ||
| "regexp" | ||
| "strings" | ||
| ) | ||
|
|
||
| var urlPathPatt = regexp.MustCompile(`(?i)(?:/)(?P<fragment>[^/\s]*)`) | ||
| var biliVideoIdPatt = regexp.MustCompile(`(?i)^((?:av|ep)(?:\d+)|bv(?:[a-zA-Z0-9]+))$`) | ||
|
|
||
| var biliDomains = []string{ | ||
| "b23.tv", | ||
| "bilibili.com", | ||
| "www.bilibili.com", | ||
| "space.bilibili.com", | ||
| "m.bilibili.com", | ||
| "t.bilibili.com", | ||
| "live.bilibili.com", | ||
| } | ||
|
|
||
| var biliRetainQueryParams = []string{ | ||
| "p", | ||
| "t", | ||
| "tab", | ||
| } | ||
|
|
||
| func clearBiliUrlQuery(u *urlx.ExtraUrl) error { | ||
| q, err := removeBiliTracingParramFromQuery(u.Query) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| u.Query = q | ||
| return nil | ||
| } | ||
|
|
||
| func removeBiliTracingParramFromQuery(query string) (string, error) { | ||
| if query == "" { | ||
| return "", nil | ||
| } | ||
|
|
||
| if query[0] == '?' { | ||
| query = query[1:] | ||
| } | ||
|
|
||
| old, err := url.ParseQuery(query) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| newMap := make(url.Values) | ||
| for _, k := range biliRetainQueryParams { | ||
| if v, ok := old[k]; ok { | ||
| newMap[k] = v | ||
| } | ||
| } | ||
|
|
||
| ret := newMap.Encode() | ||
| if ret != "" { | ||
| ret = "?" + ret | ||
| } | ||
| return ret, nil | ||
| } | ||
|
|
||
| func writeBiliUrl(buf *bytes.Buffer, u *urlx.ExtraUrl) error { | ||
| if strings.ToLower(u.Domain) == "b23.tv" { | ||
| to, err := processB23Url(context.TODO(), u) | ||
| if err != nil { | ||
| return nil | ||
| } | ||
| buf.WriteString(to) | ||
| } else { | ||
| err := clearBiliUrlQuery(u) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| buf.WriteString(u.StringByFields()) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func processB23Url(ctx context.Context, u *urlx.ExtraUrl) (string, error) { | ||
| path := u.Path | ||
| pathFragm := spliteUrlPath(path) | ||
| if len(pathFragm) == 0 { | ||
| if u.Query == "" { | ||
| return u.Text, nil | ||
| } | ||
| err := clearBiliUrlQuery(u) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return u.StringByFields(), nil | ||
| } | ||
|
|
||
| // process origin video URL | ||
| firstFr := pathFragm[0] | ||
| if biliVideoIdPatt.MatchString(firstFr) { | ||
| u.Path = "/" + firstFr | ||
| err := clearBiliUrlQuery(u) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return u.StringByFields(), nil | ||
| } | ||
|
|
||
| // process short video URL | ||
| return processBiliShortenUrl(ctx, u) | ||
| } | ||
|
|
||
| func processBiliShortenUrl(ctx context.Context, u *urlx.ExtraUrl) (string, error) { | ||
| oriUrl := u.Text | ||
| req, err := http.NewRequestWithContext(ctx, http.MethodGet, oriUrl, nil) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| client := http.Client{ | ||
| CheckRedirect: func(req *http.Request, via []*http.Request) error { | ||
| return http.ErrUseLastResponse | ||
| }, | ||
| } | ||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| // get origin URL from a shorten URL | ||
| if resp.StatusCode >= 300 && resp.StatusCode < 400 { | ||
| to, err := resp.Location() | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| e := urlx.UrlToExtraUrl(to) | ||
|
|
||
| // video URL without `p` and `t` query params | ||
| // use `b23.tv` domain for shorten URL | ||
| if strings.HasPrefix(e.Path, "/video/") { | ||
| pQ := to.Query().Get("p") | ||
| tQ := to.Query().Get("t") | ||
| paths := spliteUrlPath(e.Path) | ||
| if len(paths) >= 2 && (pQ == "" || pQ == "1") && tQ == "" { | ||
| e.Path = "/" + paths[1] | ||
| e.Domain = "b23.tv" | ||
| e.Query = "" | ||
| } | ||
| } | ||
| err = clearBiliUrlQuery(e) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return e.StringByFields(), nil | ||
| } | ||
|
|
||
| return u.Text, nil | ||
| } | ||
|
|
||
| func spliteUrlPath(path string) []string { | ||
| matches := urlPathPatt.FindAllStringSubmatchIndex(path, -1) | ||
|
|
||
| ret := make([]string, 0, len(matches)) | ||
| for _, m := range matches { | ||
| ret = append(ret, urlx.SubmatchGroupStringByName(urlPathPatt, path, m, "fragment")) | ||
| } | ||
| return ret | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package inline | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "csust-got/util/urlx" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func Test_writeBiliUrl(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| url string | ||
| want string | ||
| wantErr bool | ||
| }{ | ||
| } | ||
|
|
||
| buf := bytes.NewBufferString("") | ||
| for _, tt := range tests { | ||
| buf.Reset() | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| u := urlx.ExtractStr(tt.url)[0] | ||
| err := writeBiliUrl(buf, u.Url) | ||
| if (err != nil) != tt.wantErr { | ||
| t.Errorf("writeBiliUrl() error = %v, wantErr %v", err, tt.wantErr) | ||
| return | ||
| } | ||
| if err == nil { | ||
| assert.Equal(t, tt.want, buf.String()) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package inline | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "csust-got/config" | ||
| "csust-got/log" | ||
| "csust-got/util" | ||
| "csust-got/util/urlx" | ||
| "errors" | ||
| "regexp" | ||
| "slices" | ||
| "strings" | ||
|
|
||
| "go.uber.org/zap" | ||
| tb "gopkg.in/telebot.v3" | ||
| ) | ||
|
|
||
| //nolint:revive // It's too long. | ||
| var biliUrlRegex = `(?i)((?P<schema>https?://)?(?P<host>(?P<sub_domain>[\w\d\-]+\.)?(?P<main_domain>b23\.tv|bilibili\.com))(?P<path>(?:/[^\s\?#]*)*)?(?P<query>\?[^\s#]*)?(?P<hash>#[\S]*)?)` | ||
| var biliPatt = regexp.MustCompile(biliUrlRegex) | ||
|
|
||
| var ( | ||
| // ErrContextCanceled is returned when context is canceled | ||
| ErrContextCanceled = errors.New("context canceled") | ||
| ) | ||
|
|
||
| func init() { | ||
| biliPatt.Longest() | ||
| } | ||
|
|
||
| // RegisterInlineHandler regiester inline mode handler | ||
| func RegisterInlineHandler(bot *tb.Bot, conf *config.Config) { | ||
| bot.Handle(tb.OnQuery, handler(conf)) | ||
| } | ||
|
|
||
| func handler(conf *config.Config) func(ctx tb.Context) error { | ||
| return func(ctx tb.Context) error { | ||
| q := ctx.Query() | ||
| text := q.Text | ||
|
|
||
| exs := urlx.ExtractStr(text) | ||
| log.Debug("extracted urls", zap.String("origin", text), zap.Any("urls", exs)) | ||
|
|
||
| buf := bytes.NewBufferString("") | ||
| err := writeAll(buf, exs) | ||
| if err != nil { | ||
| log.Error("write all error", zap.Error(err)) | ||
| return err | ||
| } | ||
|
|
||
| reText := buf.String() | ||
| log.Debug("replaced text", zap.String("origin", text), zap.String("replaced", reText)) | ||
| reTextEscaped := util.EscapeTelegramReservedChars(reText) | ||
| err = ctx.Answer(&tb.QueryResponse{ | ||
| Results: tb.Results{ | ||
| &tb.ArticleResult{ | ||
| ResultBase: tb.ResultBase{ | ||
| ParseMode: tb.ModeMarkdownV2, | ||
| }, | ||
| Title: "ει", | ||
| Description: reText, | ||
| Text: reTextEscaped, | ||
| }, | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| log.Error("inline mode answer error", zap.Error(err)) | ||
| } | ||
| return nil | ||
| } | ||
| } | ||
|
|
||
| func writeAll(buf *bytes.Buffer, exs []*urlx.Extra) error { | ||
| for _, e := range exs { | ||
| if e.Type == urlx.TypeUrl { | ||
| err := writeUrl(buf, e) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } else { | ||
| buf.WriteString(e.Text) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func writeUrl(buf *bytes.Buffer, e *urlx.Extra) error { | ||
| u := e.Url | ||
|
|
||
| if slices.Contains(biliDomains, strings.ToLower(u.Domain)) { | ||
| err := writeBiliUrl(buf, u) | ||
| return err | ||
| } | ||
|
|
||
| buf.WriteString(u.Text) | ||
| return nil | ||
| } |
Oops, something went wrong.