DOP-2549-2: Absolute minimal Search infrastructure POC (#592)

* DOP-2549-2: Absolute minimal POC

* DOP-2549-2: Add python script, change Job classes to types

* DOP-2549-2: Move Job fields, integrate tests

* DOP-2549-2: Remove unrelated import error

* DOP-2549-2: Fix Dockerfile COPY location

* DOP-2549-2: Add missing await

* DOP-2549-2: Add tests

* DOP-2549-2: Rearchitecture

* DOP-2549-2: Add forgotten type

* DOP-2549-2: Add missing JobStatus import

Author: mango-db

src/entities/job.ts

@@ -1,32 +1,21 @@
 // TODO: Cut down on null and undefined type definition allowances
+
 export enum JobStatus {
   inQueue = 'inQueue',
   inProgress = 'inProgress',
   completed = 'completed',
   failed = 'failed',
 }
 
-export interface IJob {
-  _id: string;
-  payload: IPayload;
-  createdTime: Date;
-  email: string;
-  endTime: Date | null | undefined;
-  error: any | null | undefined;
-  logs: string[] | null | undefined;
-  priority: number | null | undefined;
-  result: any | null | undefined;
-  startTime: Date;
-  status: string;
-  title: string;
-  user: string;
-  comMessage: string[] | null | undefined;
-  purgedUrls: string[] | null | undefined;
-  buildCommands: Array<string>;
-  deployCommands: Array<string>;
-}
+// TODO: Formalize JobTypes
+// export enum JobType {
+//   githubPush = 'githubPush',
+//   manifestGeneration = 'manifestGeneration',
+//   productionDeploy = 'productionDeploy',
+//   regression = 'regression',
+// }
 
-export interface IPayload {
+export type Payload = {
   jobType: string;
   source: string;
   action: string;
@@ -53,28 +42,28 @@ export interface IPayload {
   prefix: string;
   project: string;
   includeInGlobalSearch: boolean;
-}
+};
 
-export class Job implements IJob {
+export type Job = {
   _id: string;
-  payload: IPayload;
+  payload: Payload;
   createdTime: Date;
-  email: string;
   endTime: Date | null | undefined;
-  error: any;
+  error: any | null | undefined;
   logs: string[] | null | undefined;
   priority: number | null | undefined;
-  result: any;
+  result: any | null | undefined;
   startTime: Date;
-  status: string;
+  status: JobStatus | null;
   title: string;
   user: string;
-  comMessage: string[] | null | undefined;
-  purgedUrls: string[] | null | undefined;
   manifestPrefix: string | undefined;
   pathPrefix: string | null | undefined;
   mutPrefix: string | null | undefined;
   buildCommands: string[];
   deployCommands: string[];
-  invalidationStatusURL: string | null | undefined;
-}
+  email: string; // probably can be removed
+  comMessage: string[] | null | undefined;
+  purgedUrls: string[] | null | undefined;
+  shouldGenerateSearchManifest: boolean;
+};

src/job/jobHandler.ts

@@ -1,4 +1,4 @@
-import { IJob } from '../entities/job';
+import type { Payload, Job } from '../entities/job';
 import { JobRepository } from '../repositories/jobRepository';
 import { RepoBranchesRepository } from '../repositories/repoBranchesRepository';
 import { ICDNConnector } from '../services/cdn';
@@ -12,8 +12,8 @@ import { IJobValidator } from './jobValidator';
 require('fs');
 
 export abstract class JobHandler {
-  private _currJob: IJob;
-  public get currJob(): IJob {
+  private _currJob: Job;
+  public get currJob(): Job {
     return this._currJob;
   }
 
@@ -55,7 +55,7 @@ export abstract class JobHandler {
   protected _repoBranchesRepo: RepoBranchesRepository;
 
   constructor(
-    job: IJob,
+    job: Job,
     config: IConfig,
     jobRepository: JobRepository,
     fileSystemServices: IFileSystemServices,
@@ -414,6 +414,54 @@ export abstract class JobHandler {
     }
   }
 
+  // Creates and pushes the related manifestJob
+  @throwIfJobInterupted()
+  public async queueManifestJob(): Promise<void> {
+    // Rudimentary error prevention
+    if (this._currJob.payload.jobType.includes('manifestGeneration')) {
+      this._logger.error(
+        this._currJob._id,
+        `Incorrectly attempted to queue another search manifest job 
+      based on known ManifestJob '${this._currJob._id}'. Please contact documentation platform team.`
+      );
+      return;
+    }
+    const manifestPayload: Payload = this._currJob.payload;
+    manifestPayload.jobType = 'manifestGeneration';
+    const manifestJob: Job = {
+      _id: '',
+      payload: manifestPayload,
+      createdTime: new Date(),
+      endTime: undefined,
+      error: undefined,
+      logs: undefined,
+      priority: 2,
+      result: undefined,
+      startTime: new Date(),
+      status: null,
+      title: this._currJob.title + ' - search manifest generation',
+      user: this._currJob.user,
+      manifestPrefix: this._currJob.manifestPrefix,
+      pathPrefix: this._currJob.pathPrefix,
+      mutPrefix: this._currJob.mutPrefix,
+      buildCommands: [],
+      deployCommands: [],
+      email: '',
+      comMessage: null,
+      purgedUrls: null,
+      shouldGenerateSearchManifest: false,
+    };
+    try {
+      await this._jobRepository.insertJob(manifestJob, this._config.get('jobsQueueUrl'));
+    } catch (error) {
+      this._logger.error(
+        manifestJob._id,
+        `Failed to queue search manifest job for build job '${this._currJob._id}'. 
+      Error: ${error.message}. Search results for this property will not be updated.`
+      );
+    }
+  }
+
   public stop(): void {
     this._shouldStop = true;
   }

src/job/jobManager.ts

@@ -2,21 +2,29 @@ import { IJobValidator } from './jobValidator';
 import { ICDNConnector } from '../services/cdn';
 import { ProductionJobHandler } from './productionJobHandler';
 import { RegressionJobHandler } from './regressionJobHandler';
+import { ManifestJobHandler } from './manifestJobHandler';
 import { StagingJobHandler } from './stagingJobHandler';
 import { IRepoConnector } from '../services/repo';
 import { IJobRepoLogger } from '../services/logger';
 import { JobHandler } from './jobHandler';
 import { IJobCommandExecutor } from '../services/commandExecutor';
 import { InvalidJobError } from '../errors/errors';
-import { IJob } from '../entities/job';
+import { Job } from '../entities/job';
 import { JobRepository } from '../repositories/jobRepository';
 import { IFileSystemServices } from '../services/fileServices';
 import { IConfig } from 'config';
 import { RepoBranchesRepository } from '../repositories/repoBranchesRepository';
 
+export const jobHandlerMap = {
+  githubPush: StagingJobHandler,
+  manifestGeneration: ManifestJobHandler,
+  productionDeploy: ProductionJobHandler,
+  regression: RegressionJobHandler,
+};
+
 export class JobHandlerFactory {
   public createJobHandler(
-    job: IJob,
+    job: Job,
     config: IConfig,
     jobRepository: JobRepository,
     fileSystemServices: IFileSystemServices,
@@ -27,34 +35,9 @@ export class JobHandlerFactory {
     validator: IJobValidator,
     repoBranchesRepo: RepoBranchesRepository
   ): JobHandler {
-    if (job.payload.jobType === 'regression') {
-      return new RegressionJobHandler(
-        job,
-        config,
-        jobRepository,
-        fileSystemServices,
-        commandExecutor,
-        cdnConnector,
-        repoConnector,
-        logger,
-        validator,
-        repoBranchesRepo
-      );
-    } else if (job.payload.jobType === 'githubPush') {
-      return new StagingJobHandler(
-        job,
-        config,
-        jobRepository,
-        fileSystemServices,
-        commandExecutor,
-        cdnConnector,
-        repoConnector,
-        logger,
-        validator,
-        repoBranchesRepo
-      );
-    } else if (job.payload.jobType === 'productionDeploy') {
-      return new ProductionJobHandler(
+    const jt = job.payload?.jobType;
+    if (jt in jobHandlerMap) {
+      return new jobHandlerMap[jt](
         job,
         config,
         jobRepository,
@@ -129,7 +112,7 @@ export class JobManager {
     return this._shouldStop;
   }
 
-  async workEx(job: IJob): Promise<void> {
+  async workEx(job: Job): Promise<void> {
     try {
       this._jobHandler = null;
       if (job?.payload) {
@@ -145,21 +128,21 @@ export class JobManager {
     }
   }
 
-  async getQueuedJob(): Promise<IJob | null> {
+  async getQueuedJob(): Promise<Job | null> {
     return await this._jobRepository.getOneQueuedJobAndUpdate().catch((error) => {
       this._logger.error('JobManager', `Error: ${error}`);
       return null;
     });
   }
 
-  async getJob(jobId: string): Promise<IJob | null> {
+  async getJob(jobId: string): Promise<Job | null> {
     return await this._jobRepository.getJobByIdAndUpdate(jobId).catch((error) => {
       this._logger.error('JobManager', `Error: ${error}`);
       return null;
     });
   }
 
-  async createHandlerAndExecute(job: IJob): Promise<void> {
+  async createHandlerAndExecute(job: Job): Promise<void> {
     this._jobHandler = this._jobHandlerFactory.createJobHandler(
       job,
       this._config,

src/job/jobValidator.ts

@@ -1,15 +1,15 @@
 import { AuthorizationError, InvalidJobError } from '../errors/errors';
 import validator from 'validator';
-import { IJob } from '../entities/job';
+import type { Job } from '../entities/job';
 import { IFileSystemServices } from '../services/fileServices';
 import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository';
 import { RepoBranchesRepository } from '../repositories/repoBranchesRepository';
 
 export interface IJobValidator {
-  throwIfJobInvalid(job: IJob): Promise<void>;
-  throwIfBranchNotConfigured(job: IJob): Promise<void>;
-  throwIfUserNotEntitled(job: IJob): Promise<void>;
-  throwIfNotPublishable(job: IJob): void;
+  throwIfJobInvalid(job: Job): Promise<void>;
+  throwIfBranchNotConfigured(job: Job): Promise<void>;
+  throwIfUserNotEntitled(job: Job): Promise<void>;
+  throwIfNotPublishable(job: Job): void;
 }
 
 export class JobValidator implements IJobValidator {
@@ -26,21 +26,21 @@ export class JobValidator implements IJobValidator {
     this._repoBranchesRepository = repoBranchesRepository;
   }
 
-  async throwIfUserNotEntitled(job: IJob): Promise<void> {
+  async throwIfUserNotEntitled(job: Job): Promise<void> {
     const entitlementsObject = await this._repoEntitlementRepository.getRepoEntitlementsByGithubUsername(job.user);
     if (!entitlementsObject?.repos?.includes(`${job.payload.repoOwner}/${job.payload.repoName}`)) {
       throw new AuthorizationError(`${job.user} is not entitled for repo ${job.payload.repoName}`);
     }
   }
 
-  async throwIfBranchNotConfigured(job: IJob): Promise<void> {
+  async throwIfBranchNotConfigured(job: Job): Promise<void> {
     job.payload.repoBranches = await this._repoBranchesRepository.getRepoBranchesByRepoName(job.payload.repoName);
     if (!job.payload?.repoBranches) {
       throw new AuthorizationError(`repoBranches not found for ${job.payload.repoName}`);
     }
   }
 
-  throwIfNotPublishable(job: IJob): void {
+  throwIfNotPublishable(job: Job): void {
     let found = false;
     if (job?.payload?.repoBranches) {
       job.payload.repoBranches['branches'].forEach((branch) => {
@@ -55,7 +55,7 @@ export class JobValidator implements IJobValidator {
     }
   }
 
-  public async throwIfJobInvalid(job: IJob): Promise<void> {
+  public async throwIfJobInvalid(job: Job): Promise<void> {
     this._validateInput(job);
     if (this.isProd(job.payload.jobType)) {
       await this.throwIfUserNotEntitled(job);
@@ -66,7 +66,7 @@ export class JobValidator implements IJobValidator {
     return jobType === 'productionDeploy';
   }
 
-  private _validateInput(job: IJob): void {
+  private _validateInput(job: Job): void {
     if (!job.payload.project) {
       throw new InvalidJobError('Invalid project');
     }

src/job/manifestJobHandler.ts

@@ -0,0 +1,65 @@
+import { JobHandler } from './jobHandler';
+import { IConfig } from 'config';
+import type { Job } from '../entities/job';
+import { JobRepository } from '../repositories/jobRepository';
+import { ICDNConnector } from '../services/cdn';
+import { CommandExecutorResponse, IJobCommandExecutor } from '../services/commandExecutor';
+import { IFileSystemServices } from '../services/fileServices';
+import { IJobRepoLogger } from '../services/logger';
+import { IRepoConnector } from '../services/repo';
+import { IJobValidator } from './jobValidator';
+import { RepoBranchesRepository } from '../repositories/repoBranchesRepository';
+
+export class ManifestJobHandler extends JobHandler {
+  constructor(
+    job: Job,
+    config: IConfig,
+    jobRepository: JobRepository,
+    fileSystemServices: IFileSystemServices,
+    commandExecutor: IJobCommandExecutor,
+    cdnConnector: ICDNConnector,
+    repoConnector: IRepoConnector,
+    logger: IJobRepoLogger,
+    validator: IJobValidator,
+    repoBranchesRepo: RepoBranchesRepository
+  ) {
+    super(
+      job,
+      config,
+      jobRepository,
+      fileSystemServices,
+      commandExecutor,
+      cdnConnector,
+      repoConnector,
+      logger,
+      validator,
+      repoBranchesRepo
+    );
+    this.name = 'Manifest';
+  }
+
+  // TODO: Make this a non-state-mutating function, e.g. return the deployCommands?
+  prepDeployCommands(): void {
+    this.currJob.deployCommands = [
+      '. /venv/bin/activate',
+      `cd repos/${this.currJob.payload.repoName}`,
+      'echo IGNORE: testing manifest generation deploy commands',
+      'python3 test-mut-script.py',
+    ];
+  }
+
+  prepStageSpecificNextGenCommands(): void {
+    return;
+  }
+
+  async deploy(): Promise<CommandExecutorResponse> {
+    try {
+      const resp = await this.deployGeneric(); // runs prepDeployCommands
+      await this.logger.save(this.currJob._id, `(generate manifest) Manifest generation details:\n\n${resp?.output}`);
+      return resp;
+    } catch (errResult) {
+      await this.logger.save(this.currJob._id, `(generate manifest) stdErr: ${errResult.stderr}`);
+      throw errResult;
+    }
+  }
+}