Fetch token rates in batches of 100 (#3650)

In cases where users have an enormous amount of tokens, we don't want to
slow down our Price API with large requests. To combat this, when
fetching token rates, send requests in batches of 100.

Author: mcmire

packages/assets-controllers/CHANGELOG.md

@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **BREAKING:** Change signature of `TokenRatesController.fetchAndMapExchangeRates` ([#3600](https://github.com/MetaMask/core/pull/3600))
   - This method now takes an object with shape `{ tokenContractAddresses: Hex[]; chainId: Hex; nativeCurrency: string; }` rather than positional arguments
 - Update TokenListController to fetch prefiltered set of tokens from the API, reducing response data and removing the need for filtering logic ([#2054](https://github.com/MetaMask/core/pull/2054))
+- Update TokenRatesController to request token rates from the Price API in batches of 100 ([#3650](https://github.com/MetaMask/core/pull/3650))
 
 ### Removed
 - **BREAKING:** Remove `fetchExchangeRate` method from TokenRatesController ([#3600](https://github.com/MetaMask/core/pull/3600))

packages/assets-controllers/src/TokenRatesController.test.ts

@@ -1,7 +1,12 @@
-import { NetworksTicker, toHex } from '@metamask/controller-utils';
+import {
+  NetworksTicker,
+  toChecksumHexAddress,
+  toHex,
+} from '@metamask/controller-utils';
 import type { NetworkState } from '@metamask/network-controller';
 import type { PreferencesState } from '@metamask/preferences-controller';
 import type { Hex } from '@metamask/utils';
+import { add0x } from '@metamask/utils';
 import nock from 'nock';
 import { useFakeTimers } from 'sinon';
 
@@ -13,7 +18,7 @@ import type {
 } from './token-prices-service/abstract-token-prices-service';
 import type { TokenBalancesState } from './TokenBalancesController';
 import { TokenRatesController } from './TokenRatesController';
-import type { TokenRatesConfig } from './TokenRatesController';
+import type { TokenRatesConfig, Token } from './TokenRatesController';
 import type { TokensState } from './TokensController';
 
 const defaultSelectedAddress = '0x0000000000000000000000000000000000000001';
@@ -1682,6 +1687,58 @@ describe('TokenRatesController', () => {
       );
     });
 
+    it('fetches rates for all tokens in batches of 100', async () => {
+      const chainId = toHex(1);
+      const ticker = 'ETH';
+      const tokenAddresses = [...new Array(200).keys()]
+        .map(buildAddress)
+        .sort();
+      const tokenPricesService = buildMockTokenPricesService({
+        fetchTokenPrices: fetchTokenPricesWithIncreasingPriceForEachToken,
+      });
+      const fetchTokenPricesSpy = jest.spyOn(
+        tokenPricesService,
+        'fetchTokenPrices',
+      );
+      const tokens = tokenAddresses.map((tokenAddress) => {
+        return buildToken({ address: tokenAddress });
+      });
+      await withController(
+        {
+          options: {
+            ticker,
+            tokenPricesService,
+          },
+        },
+        async ({ controller, controllerEvents }) => {
+          await callUpdateExchangeRatesMethod({
+            allTokens: {
+              [chainId]: {
+                [controller.config.selectedAddress]: tokens,
+              },
+            },
+            chainId,
+            controller,
+            controllerEvents,
+            method,
+            nativeCurrency: ticker,
+          });
+
+          expect(fetchTokenPricesSpy).toHaveBeenCalledTimes(2);
+          expect(fetchTokenPricesSpy).toHaveBeenNthCalledWith(1, {
+            chainId,
+            tokenContractAddresses: tokenAddresses.slice(0, 100),
+            currency: ticker,
+          });
+          expect(fetchTokenPricesSpy).toHaveBeenNthCalledWith(2, {
+            chainId,
+            tokenContractAddresses: tokenAddresses.slice(100),
+            currency: ticker,
+          });
+        },
+      );
+    });
+
     it('updates all rates', async () => {
       const tokenAddresses = [
         '0x0000000000000000000000000000000000000001',
@@ -1900,6 +1957,68 @@ describe('TokenRatesController', () => {
       );
     });
 
+    it('fetches rates for all tokens in batches of 100 when native currency is not supported by the Price API', async () => {
+      const chainId = toHex(1);
+      const ticker = 'UNSUPPORTED';
+      const tokenAddresses = [...new Array(200).keys()]
+        .map(buildAddress)
+        .sort();
+      const tokenPricesService = buildMockTokenPricesService({
+        fetchTokenPrices: fetchTokenPricesWithIncreasingPriceForEachToken,
+        validateCurrencySupported: (currency: unknown): currency is string => {
+          return currency !== ticker;
+        },
+      });
+      const fetchTokenPricesSpy = jest.spyOn(
+        tokenPricesService,
+        'fetchTokenPrices',
+      );
+      const tokens = tokenAddresses.map((tokenAddress) => {
+        return buildToken({ address: tokenAddress });
+      });
+      nock('https://min-api.cryptocompare.com')
+        .get('/data/price')
+        .query({
+          fsym: 'ETH',
+          tsyms: ticker,
+        })
+        .reply(200, { [ticker]: 0.5 });
+      await withController(
+        {
+          options: {
+            ticker,
+            tokenPricesService,
+          },
+        },
+        async ({ controller, controllerEvents }) => {
+          await callUpdateExchangeRatesMethod({
+            allTokens: {
+              [chainId]: {
+                [controller.config.selectedAddress]: tokens,
+              },
+            },
+            chainId,
+            controller,
+            controllerEvents,
+            method,
+            nativeCurrency: ticker,
+          });
+
+          expect(fetchTokenPricesSpy).toHaveBeenCalledTimes(2);
+          expect(fetchTokenPricesSpy).toHaveBeenNthCalledWith(1, {
+            chainId,
+            tokenContractAddresses: tokenAddresses.slice(0, 100),
+            currency: 'ETH',
+          });
+          expect(fetchTokenPricesSpy).toHaveBeenNthCalledWith(2, {
+            chainId,
+            tokenContractAddresses: tokenAddresses.slice(100),
+            currency: 'ETH',
+          });
+        },
+      );
+    });
+
     it('sets rates to undefined when chain is not supported by the Price API', async () => {
       const tokenAddresses = [
         '0x0000000000000000000000000000000000000001',
@@ -2284,3 +2403,32 @@ async function fetchTokenPricesWithIncreasingPriceForEachToken<
     };
   }, {}) as TokenPricesByTokenContractAddress<TokenAddress, Currency>;
 }
+
+/**
+ * Constructs a checksum Ethereum address.
+ *
+ * @param number - The address as a decimal number.
+ * @returns The address as an 0x-prefixed ERC-55 mixed-case checksum address in
+ * hexadecimal format.
+ */
+function buildAddress(number: number) {
+  return toChecksumHexAddress(add0x(number.toString(16).padStart(40, '0')));
+}
+
+/**
+ * Constructs an object that satisfies the Token interface, filling in missing
+ * properties with defaults. This makes it possible to only specify properties
+ * that the test cares about.
+ *
+ * @param overrides - The properties that should be assigned to the new token.
+ * @returns The constructed token.
+ */
+function buildToken(overrides: Partial<Token> = {}) {
+  return {
+    address: buildAddress(1),
+    decimals: 0,
+    symbol: '',
+    aggregators: [],
+    ...overrides,
+  };
+}

packages/assets-controllers/src/TokenRatesController.ts

@@ -15,6 +15,7 @@ import type { PreferencesState } from '@metamask/preferences-controller';
 import type { Hex } from '@metamask/utils';
 import { isDeepStrictEqual } from 'util';
 
+import { reduceInBatchesSerially } from './assetsUtil';
 import { fetchExchangeRate as fetchNativeCurrencyExchangeRate } from './crypto-compare';
 import type { AbstractTokenPricesService } from './token-prices-service/abstract-token-prices-service';
 import type { TokensState } from './TokensController';
@@ -95,6 +96,12 @@ export interface TokenRatesState extends BaseState {
   >;
 }
 
+/**
+ * The maximum number of token addresses that should be sent to the Price API in
+ * a single request.
+ */
+const TOKEN_PRICES_BATCH_SIZE = 100;
+
 /**
  * Uses the CryptoCompare API to fetch the exchange rate between one currency
  * and another, i.e., the multiplier to apply the amount of one currency in
@@ -505,12 +512,27 @@ export class TokenRatesController extends StaticIntervalPollingControllerV1<
     chainId: Hex;
     nativeCurrency: string;
   }): Promise<ContractExchangeRates> {
-    const tokenPricesByTokenContractAddress =
-      await this.#tokenPricesService.fetchTokenPrices({
-        tokenContractAddresses,
-        chainId,
-        currency: nativeCurrency,
-      });
+    const tokenPricesByTokenContractAddress = await reduceInBatchesSerially<
+      Hex,
+      Awaited<ReturnType<AbstractTokenPricesService['fetchTokenPrices']>>
+    >({
+      values: tokenContractAddresses,
+      batchSize: TOKEN_PRICES_BATCH_SIZE,
+      eachBatch: async (allTokenPricesByTokenContractAddress, batch) => {
+        const tokenPricesByTokenContractAddressForBatch =
+          await this.#tokenPricesService.fetchTokenPrices({
+            tokenContractAddresses: batch,
+            chainId,
+            currency: nativeCurrency,
+          });
+
+        return {
+          ...allTokenPricesByTokenContractAddress,
+          ...tokenPricesByTokenContractAddressForBatch,
+        };
+      },
+      initialResult: {},
+    });
 
     return Object.entries(tokenPricesByTokenContractAddress).reduce(
       (obj, [tokenContractAddress, tokenPrice]) => {
@@ -543,13 +565,13 @@ export class TokenRatesController extends StaticIntervalPollingControllerV1<
     nativeCurrency: string;
   }): Promise<ContractExchangeRates> {
     const [
-      tokenPricesByTokenContractAddress,
+      contractExchangeRates,
       fallbackCurrencyToNativeCurrencyConversionRate,
     ] = await Promise.all([
-      this.#tokenPricesService.fetchTokenPrices({
+      this.#fetchAndMapExchangeRatesForSupportedNativeCurrency({
         tokenContractAddresses,
-        currency: FALL_BACK_VS_CURRENCY,
         chainId: this.config.chainId,
+        nativeCurrency: FALL_BACK_VS_CURRENCY,
       }),
       getCurrencyConversionRate({
         from: FALL_BACK_VS_CURRENCY,
@@ -561,12 +583,13 @@ export class TokenRatesController extends StaticIntervalPollingControllerV1<
       return {};
     }
 
-    return Object.entries(tokenPricesByTokenContractAddress).reduce(
-      (obj, [tokenContractAddress, tokenPrice]) => {
+    return Object.entries(contractExchangeRates).reduce(
+      (obj, [tokenContractAddress, tokenValue]) => {
         return {
           ...obj,
-          [tokenContractAddress]:
-            tokenPrice.value * fallbackCurrencyToNativeCurrencyConversionRate,
+          [tokenContractAddress]: tokenValue
+            ? tokenValue * fallbackCurrencyToNativeCurrencyConversionRate
+            : undefined,
         };
       },
       {},