Use gzhttp transport when scraping data

Wikidepia · Jul 23, 2024 · 820101b · 820101b
1 parent ae140cf
commit 820101b
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 4 deletions.
diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/elastic/go-freelru v0.13.0
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/kelindar/binary v1.0.19
-	github.com/klauspost/compress v1.17.9
+	github.com/klauspost/compress v1.17.10-0.20240719110028-cfab8bd586ec
 	github.com/tdewolff/parse/v2 v2.7.15
 	github.com/tidwall/gjson v1.17.1
 	go.etcd.io/bbolt v1.3.10

diff --git a/go.sum b/go.sum
@@ -16,8 +16,8 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
 github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/kelindar/binary v1.0.19 h1:DNyQCtKjkLhBh9pnP49OWREddLB0Mho+1U/AOt/Qzxw=
 github.com/kelindar/binary v1.0.19/go.mod h1:/twdz8gRLNMffx0U4UOgqm1LywPs6nd9YK2TX52MDh8=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.10-0.20240719110028-cfab8bd586ec h1:S0mztzoHWmIpI4VuqO7MPIf3DQQxWwW5e3Uurb+HMHA=
+github.com/klauspost/compress v1.17.10-0.20240719110028-cfab8bd586ec/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=

diff --git a/handlers/scraper/data.go b/handlers/scraper/data.go
@@ -16,6 +16,7 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	"github.com/PurpleSec/escape"
 	"github.com/kelindar/binary"
+	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/zstd"
 	"github.com/tdewolff/parse/v2"
 	"github.com/tdewolff/parse/v2/js"
@@ -28,6 +29,7 @@ import (
 var (
 	timeout     = 10 * time.Second
 	ErrNotFound = errors.New("post not found")
+	transport   = gzhttp.Transport(http.DefaultTransport, gzhttp.TransportAlwaysDecompress(true))
 )
 
 var RemoteScraperAddr string
@@ -132,7 +134,7 @@ func GetData(postID string) (*InstaData, error) {
 }
 
 func (i *InstaData) ScrapeData() error {
-	client := http.Client{Timeout: timeout}
+	client := http.Client{Transport: transport, Timeout: timeout}
 
 	// Scrape from remote scraper if available
 	if len(RemoteScraperAddr) > 0 {