nb

handlers/scraper/data.go

@@ -149,6 +149,8 @@ func (i *InstaData) ScrapeData() error {
 	// Scrape from remote scraper if available
 	if err := ScrapeRemote(i); err == nil {
 		return nil
+	} else {
+		slog.Error("Failed to scrape data from remote scraper", "postID", i.PostID, "err", err)
 	}
 
 	client := http.Client{Transport: transport, Timeout: timeout}

handlers/scraper/remote.go

@@ -4,8 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync/atomic"
-	"syscall"
 	"time"
 
 	"github.com/kelindar/binary"
@@ -17,11 +15,9 @@ type remoteResult struct {
 	outChan   chan error
 }
 
-var connStream atomic.Int32
 var inChan chan remoteResult
 
 func init() {
-	connStream.Store(0)
 	inChan = make(chan remoteResult)
 
 	ln, err := net.Listen("tcp", "localhost:4444")
@@ -54,34 +50,25 @@ func handleConnection(conn net.Conn) {
 			break
 		}
 
-		buf := make([]byte, 64*1024)
 		go func() {
-			connStream.Add(1)
-			defer connStream.Add(-1)
 			for {
 				select {
 				case rm := <-inChan:
-					buf = []byte(rm.instaData.PostID)
+					buf := []byte(rm.instaData.PostID)
 					if _, err = stream.Write(buf); err != nil {
-						if errors.Is(err, syscall.EPIPE) {
-							stream.Close()
-						}
 						rm.outChan <- err
 						return
 					}
 
-					var n int
-					outBuf := make([]byte, 128*1024)
-					if n, err = stream.Read(outBuf); err != nil {
-						if errors.Is(err, syscall.EPIPE) {
-							stream.Close()
-						}
+					outBuf := make([]byte, 1024*1024)
+					if n, err := stream.Read(outBuf); err == nil {
+						err = binary.Unmarshal(outBuf[:n], rm.instaData)
+						rm.outChan <- err
+					} else {
 						rm.outChan <- err
 						return
 					}
 
-					err = binary.Unmarshal(outBuf[:n], rm.instaData)
-					rm.outChan <- err
 				case <-stream.GetDieCh():
 					return
 				}
@@ -91,10 +78,6 @@ func handleConnection(conn net.Conn) {
 }
 
 func ScrapeRemote(i *InstaData) error {
-	if connStream.Load() == 0 {
-		return errors.New("Remote scraper is not running")
-	}
-
 	remoteRes := remoteResult{
 		instaData: i,
 		outChan:   make(chan error),
@@ -103,7 +86,7 @@ func ScrapeRemote(i *InstaData) error {
 	select {
 	case inChan <- remoteRes:
 	default:
-		return nil
+		return errors.New("remote scraper is not running")
 	}
 
 	select {