Skip to content

Commit

Permalink
fix: skip unneccessary browser context generation for api scrapers
Browse files Browse the repository at this point in the history
Signed-off-by: Eiko Wagenknecht <[email protected]>
  • Loading branch information
eikowagenknecht committed Feb 9, 2025
1 parent c278d16 commit 346b703
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 23 deletions.
9 changes: 1 addition & 8 deletions src/services/scraper/base/scraper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,6 @@ export abstract class BaseScraper {
* const offers = await scraper.scrape();
*/
public async scrape(): Promise<NewOffer[]> {
this.context = browserService.getContext();

try {
const offers = await this.readOffers();
const cleanedOffers = this.cleanOffers(offers);
Expand Down Expand Up @@ -189,12 +187,7 @@ export abstract class BaseScraper {
pageReadySelector: string;
pageLoadedHook?: (page: Page) => Promise<void>;
}): Promise<Omit<NewOffer, "category">[]> {
if (!this.context) {
throw new ScraperError(
"Browser context not initialized. Call initialize() first.",
this.getScraperName(),
);
}
this.context = browserService.getContext();

const offers: Omit<NewOffer, "category">[] = [];
let page: Page | null = null;
Expand Down
20 changes: 5 additions & 15 deletions src/services/scraper/implementations/epicApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import {
OfferType,
} from "@/types/basic";
import type { NewOffer } from "@/types/database";
import { logger } from "@/utils/logger";
import {
ApolloClient,
type DefaultOptions,
Expand Down Expand Up @@ -164,21 +163,12 @@ export class EpicGamesApiScraper extends BaseScraper {

override async readOffers(): Promise<Omit<NewOffer, "category">[]> {
const client = this.createClient();
try {
const response = await client.query<CatalogData, { count: number }>({
query: FREEGAMES_QUERY,
variables: { count: 1000 },
});
const response = await client.query<CatalogData, { count: number }>({
query: FREEGAMES_QUERY,
variables: { count: 1000 },
});

return this.parseOffers(response.data);
} catch (error) {
if (error instanceof Error) {
logger.error("Error fetching free games:", error.message);
} else {
logger.error("Unknown error occurred while fetching free games");
}
return [];
}
return this.parseOffers(response.data);
}

protected override shouldAlwaysHaveOffers(): boolean {
Expand Down

0 comments on commit 346b703

Please sign in to comment.