URL Store

.github/workflows/main.yaml

@@ -10,6 +10,7 @@ on: [push]
 env:
   CI_BUILD_NUM: ${{ github.run_id }}
   CI_BRANCH: ${{ github.ref_name }}
+  NODE_OPTIONS: --max-old-space-size=4096
 
 jobs:
   test:

packages/spacecat-shared-data-access/src/models/audit-url/audit-url.collection.js

@@ -0,0 +1,325 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { hasText } from '@adobe/spacecat-shared-utils';
+
+import BaseCollection from '../base/base.collection.js';
+
+/**
+ * AuditUrlCollection - A collection class responsible for managing AuditUrl entities.
+ * Extends the BaseCollection to provide specific methods for interacting with AuditUrl records.
+ *
+ * @class AuditUrlCollection
+ * @extends BaseCollection
+ */
+class AuditUrlCollection extends BaseCollection {
+  /**
+   * Sorts audit URLs by a specified field.
+   * @param {Array} auditUrls - Array of AuditUrl objects to sort.
+   * @param {string} sortBy - Field to sort by ('rank', 'traffic', 'url', 'createdAt', 'updatedAt').
+   * @param {string} sortOrder - Sort order ('asc' or 'desc'). Default: 'asc'.
+   * @returns {Array} Sorted array of AuditUrl objects.
+   * @private
+   */
+  static sortAuditUrls(auditUrls, sortBy = 'rank', sortOrder = 'asc') {
+    if (!auditUrls || auditUrls.length === 0) {
+      return auditUrls;
+    }
+
+    const sorted = [...auditUrls].sort((a, b) => {
+      let aValue;
+      let bValue;
+
+      // Get values using getter methods if available
+      switch (sortBy) {
+        case 'rank':
+          aValue = a.getRank ? a.getRank() : a.rank;
+          bValue = b.getRank ? b.getRank() : b.rank;
+          break;
+        case 'traffic':
+          aValue = a.getTraffic ? a.getTraffic() : a.traffic;
+          bValue = b.getTraffic ? b.getTraffic() : b.traffic;
+          break;
+        case 'url':
+          aValue = a.getUrl ? a.getUrl() : a.url;
+          bValue = b.getUrl ? b.getUrl() : b.url;
+          break;
+        case 'createdAt':
+          aValue = a.getCreatedAt ? a.getCreatedAt() : a.createdAt;
+          bValue = b.getCreatedAt ? b.getCreatedAt() : b.createdAt;
+          break;
+        case 'updatedAt':
+          aValue = a.getUpdatedAt ? a.getUpdatedAt() : a.updatedAt;
+          bValue = b.getUpdatedAt ? b.getUpdatedAt() : b.updatedAt;
+          break;
+        default:
+          return 0;
+      }
+
+      // Handle null/undefined values (push to end)
+      if (aValue == null && bValue == null) return 0;
+      if (aValue == null) return 1;
+      if (bValue == null) return -1;
+
+      // Compare values
+      let comparison = 0;
+      if (typeof aValue === 'string' && typeof bValue === 'string') {
+        comparison = aValue.localeCompare(bValue);
+      } else if (aValue < bValue) {
+        comparison = -1;
+      } else if (aValue > bValue) {
+        comparison = 1;
+      } else {
+        comparison = 0;
+      }
+
+      return sortOrder === 'desc' ? -comparison : comparison;
+    });
+
+    return sorted;
+  }
+
+  /**
+   * Finds an audit URL by site ID and URL.
+   * This is a convenience method for looking up a specific URL.
+   *
+   * @param {string} siteId - The site ID.
+   * @param {string} url - The URL to find.
+   * @returns {Promise<AuditUrl|null>} The found AuditUrl or null.
+   */
+  async findBySiteIdAndUrl(siteId, url) {
+    if (!hasText(siteId) || !hasText(url)) {
+      throw new Error('Both siteId and url are required');
+    }
+
+    const results = await this.allBySiteIdAndUrl(siteId, url);
+    return results.length > 0 ? results[0] : null;
+  }
+
+  /**
+   * Gets all audit URLs for a site that have a specific audit type enabled.
+   * Note: This performs filtering after retrieval since audits is an array.
+   *
+   * @param {string} siteId - The site ID.
+   * @param {string} auditType - The audit type to filter by.
+   * @param {object} [options={}] - Query options (limit, cursor, sortBy, sortOrder).
+   * @returns {Promise<{items: AuditUrl[], cursor?: string}>} Paginated results.
+   */
+  async allBySiteIdAndAuditType(siteId, auditType, options = {}) {
+    if (!hasText(siteId) || !hasText(auditType)) {
+      throw new Error('Both siteId and auditType are required');
+    }
+
+    const { sortBy, sortOrder, ...queryOptions } = options;
+
+    // Get all URLs for the site
+    const allUrls = await this.allBySiteId(siteId, queryOptions);
+
+    // Filter by audit type
+    let filtered = allUrls.filter((auditUrl) => auditUrl.isAuditEnabled(auditType));
+
+    // Apply sorting if requested
+    if (sortBy) {
+      filtered = AuditUrlCollection.sortAuditUrls(filtered, sortBy, sortOrder);
+    }
+
+    return filtered;
+  }
+
+  /**
+   * Gets all audit URLs for a site with sorting support.
+   * @param {string} siteId - The site ID.
+   * @param {object} [options={}] - Query options (limit, cursor, sortBy, sortOrder).
+   * @returns {Promise<{items: AuditUrl[], cursor?: string}>} Paginated and sorted results.
+   */
+  async allBySiteIdSorted(siteId, options = {}) {
+    if (!hasText(siteId)) {
+      throw new Error('SiteId is required');
+    }
+
+    const { sortBy, sortOrder, ...queryOptions } = options;
+
+    // Get all URLs for the site
+    const result = await this.allBySiteId(siteId, queryOptions);
+
+    // Handle both array and paginated result formats
+    const items = Array.isArray(result) ? result : (result.items || []);
+
+    // Apply sorting if requested
+    const sortedItems = sortBy
+      ? AuditUrlCollection.sortAuditUrls(items, sortBy, sortOrder) : items;
+
+    // Return in the same format as received
+    if (Array.isArray(result)) {
+      return sortedItems;
+    }
+
+    return {
+      items: sortedItems,
+      cursor: result.cursor,
+    };
+  }
+
+  /**
+   * Gets all audit URLs for a site by source with sorting support.
+   * @param {string} siteId - The site ID.
+   * @param {string} source - The source to filter by.
+   * @param {object} [options={}] - Query options (limit, cursor, sortBy, sortOrder).
+   * @returns {Promise<{items: AuditUrl[], cursor?: string}>} Paginated and sorted results.
+   */
+  async allBySiteIdAndSourceSorted(siteId, source, options = {}) {
+    if (!hasText(siteId) || !hasText(source)) {
+      throw new Error('Both siteId and source are required');
+    }
+
+    const { sortBy, sortOrder, ...queryOptions } = options;
+
+    // Get all URLs for the site and source
+    const result = await this.allBySiteIdAndSource(siteId, source, queryOptions);
+
+    // Handle both array and paginated result formats
+    const items = Array.isArray(result) ? result : (result.items || []);
+
+    // Apply sorting if requested
+    const sortedItems = sortBy
+      ? AuditUrlCollection.sortAuditUrls(items, sortBy, sortOrder) : items;
+
+    // Return in the same format as received
+    if (Array.isArray(result)) {
+      return sortedItems;
+    }
+
+    return {
+      items: sortedItems,
+      cursor: result.cursor,
+    };
+  }
+
+  /**
+   * Removes all audit URLs for a specific site.
+   * Useful for cleanup operations.
+   *
+   * @param {string} siteId - The site ID.
+   * @returns {Promise<void>}
+   */
+  async removeForSiteId(siteId) {
+    if (!hasText(siteId)) {
+      throw new Error('SiteId is required');
+    }
+
+    const urlsToRemove = await this.allBySiteId(siteId);
+    const idsToRemove = urlsToRemove.map((auditUrl) => auditUrl.getId());
+
+    if (idsToRemove.length > 0) {
+      await this.removeByIds(idsToRemove);
+    }
+  }
+
+  /**
+   * Removes audit URLs by source for a specific site.
+   * For example, remove all 'sitemap' sourced URLs.
+   *
+   * @param {string} siteId - The site ID.
+   * @param {string} source - The source to filter by.
+   * @returns {Promise<void>}
+   */
+  async removeForSiteIdAndSource(siteId, source) {
+    if (!hasText(siteId) || !hasText(source)) {
+      throw new Error('Both siteId and source are required');
+    }
+
+    const urlsToRemove = await this.allBySiteIdAndSource(siteId, source);
+    const idsToRemove = urlsToRemove.map((auditUrl) => auditUrl.getId());
+
+    if (idsToRemove.length > 0) {
+      await this.removeByIds(idsToRemove);
+    }
+  }
+
+  /**
+   * Gets all audit URLs for a site filtered by platform type with sorting support.
+   * @param {string} siteId - The site ID.
+   * @param {string} platformType - The platform type to filter by.
+   * @param {object} [options={}] - Query options (limit, cursor, sortBy, sortOrder).
+   * @returns {Promise<{items: AuditUrl[], cursor?: string}>} Paginated and sorted results.
+   */
+  async allBySiteIdAndPlatform(siteId, platformType, options = {}) {
+    if (!hasText(siteId) || !hasText(platformType)) {
+      throw new Error('Both siteId and platformType are required');
+    }
+
+    const { sortBy, sortOrder, ...queryOptions } = options;
+
+    // Use the GSI to query by siteId and platformType
+    const result = await this.queryItems(
+      { siteId, platformType },
+      queryOptions,
+      'gsi2', // The third GSI we created
+    );
+
+    // Handle both array and paginated result formats
+    const items = Array.isArray(result) ? result : (result.items || []);
+
+    // Apply sorting if requested
+    const sortedItems = sortBy
+      ? AuditUrlCollection.sortAuditUrls(items, sortBy, sortOrder) : items;
+
+    // Return in the same format as received
+    if (Array.isArray(result)) {
+      return sortedItems;
+    }
+
+    return {
+      items: sortedItems,
+      cursor: result.cursor,
+    };
+  }
+
+  /**
+   * Gets all offsite platform URLs for a site (excludes primary-site URLs).
+   * @param {string} siteId - The site ID.
+   * @param {object} [options={}] - Query options (limit, cursor, sortBy, sortOrder).
+   * @returns {Promise<{items: AuditUrl[], cursor?: string}>} Paginated and sorted offsite URLs.
+   */
+  async allOffsiteUrls(siteId, options = {}) {
+    if (!hasText(siteId)) {
+      throw new Error('SiteId is required');
+    }
+
+    const { sortBy, sortOrder, ...queryOptions } = options;
+
+    // Get all URLs for the site
+    const result = await this.allBySiteId(siteId, queryOptions);
+
+    // Handle both array and paginated result formats
+    const items = Array.isArray(result) ? result : (result.items || []);
+
+    // Filter to only offsite platform URLs
+    const offsiteItems = items.filter((url) => url.isOffsitePlatform && url.isOffsitePlatform());
+
+    // Apply sorting if requested
+    const sortedItems = sortBy
+      ? AuditUrlCollection.sortAuditUrls(offsiteItems, sortBy, sortOrder) : offsiteItems;
+
+    // Return in the same format as received
+    if (Array.isArray(result)) {
+      return sortedItems;
+    }
+
+    return {
+      items: sortedItems,
+      cursor: result.cursor,
+    };
+  }
+}
+
+export default AuditUrlCollection;