implement waitForRequest

docs/api.md

@@ -6,11 +6,12 @@ Chromeless provides TypeScript typings.
 - [`end()`](#api-end)
 
 **Chrome methods**
-- [`goto(url: string)`](#api-goto)
+- [`goto(url: string, logRequests?: boolean)`](#api-goto)
 - [`click(selector: string)`](#api-click)
 - [`wait(timeout: number)`](#api-wait-timeout)
 - [`wait(selector: string)`](#api-wait-selector)
 - [`wait(fn: (...args: any[]) => boolean, ...args: any[])`](#api-wait-fn)
+- [`waitForRequest(url: string, fn: Function)`](#api-waitForRequest)
 - [`focus(selector: string)`](#api-focus)
 - [`press(keyCode: number, count?: number, modifiers?: any)`](#api-press)
 - [`type(input: string, selector?: string)`](#api-type)
@@ -54,12 +55,13 @@ await chromeless.end()
 
 <a name="api-goto" />
 
-### goto(url: string): Chromeless<T>
+### goto(url: string, logRequests?: boolean): Chromeless<T>
 
 Navigate to a URL.
 
 __Arguments__
 - `url` - URL to navigate to
+- `logRequests` - log all requests for this session. only useful for [`waitForRequest(url: string, fn: Function)`](#api-waitForRequest)
 
 __Example__
 
@@ -138,6 +140,30 @@ await chromeless.wait(() => { return console.log('@TODO: put a better example he
 
 ---------------------------------------
 
+<a name="api-waitForRequest" />
+
+### waitForRequest(url: string, fn: (requests) => requests): Chromeless<T>
+
+Wait until one or more requests have been sent to the specified URL
+
+__Arguments__
+- `url` - All requests that include this URL are collected and passed to `fn`
+- `fn`  - Function that receives an array of all available requests as first parameter.
+          Waits until this function returns true or the timeout is reached (default: 10s)
+
+__Example__
+
+```js
+const result = await chromeless
+    .goto('https://www.google.com', true) // required to get requests before the load event is fired
+    .waitForRequest('google.com', (requests) => {
+      return requests.length > 2;
+    })
+
+console.log(result)
+```
+
+---------------------------------------
 <a name="api-focus" />
 
 ### focus(selector: string): Chromeless<T>

src/api.ts

@@ -60,8 +60,8 @@ export default class Chromeless<T extends any> implements Promise<T> {
     return this.lastReturnPromise.catch(onrejected) as Promise<U>
   }
 
-  goto(url: string): Chromeless<T> {
-    this.queue.enqueue({type: 'goto', url})
+  goto(url: string, logRequests?: boolean): Chromeless<T> {
+    this.queue.enqueue({type: 'goto', url, logRequests})
 
     return this
   }
@@ -72,6 +72,12 @@ export default class Chromeless<T extends any> implements Promise<T> {
     return this
   }
 
+  waitForRequest(url: string, fn: Function): Chromeless<T> {
+    this.lastReturnPromise = this.queue.process<string>({type: 'wait', url: url, fn: fn})
+
+    return this
+  }
+
   wait(timeout: number): Chromeless<T>
   wait(selector: string): Chromeless<T>
   wait(fn: (...args: any[]) => boolean, ...args: any[]): Chromeless<T>

src/chrome/local-runtime.ts

@@ -2,10 +2,12 @@ import * as AWS from 'aws-sdk'
 import { Client, Command, ChromelessOptions, Cookie, CookieQuery } from '../types'
 import * as cuid from 'cuid'
 import * as fs from 'fs'
+import TrafficLog from '../network'
 import {
   nodeExists,
   wait,
   waitForNode,
+  waitForRequest,
   click,
   evaluate,
   screenshot,
@@ -22,21 +24,25 @@ export default class LocalRuntime {
 
   private client: Client
   private chromlessOptions: ChromelessOptions
+  private trafficLog: TrafficLog
 
   constructor(client: Client, chromlessOptions: ChromelessOptions) {
     this.client = client
     this.chromlessOptions = chromlessOptions
+    this.trafficLog = new(TrafficLog)
   }
 
   async run(command: Command): Promise<any> {
     switch (command.type) {
       case 'goto':
-        return this.goto(command.url)
+        return this.goto(command.url, command.logRequests)
       case 'wait': {
         if (command.timeout) {
           return this.waitTimeout(command.timeout)
         } else if (command.selector) {
           return this.waitSelector(command.selector)
+        } else if (command.url) {
+          return this.waitRequest(command.url, command.fn)
         } else {
           throw new Error('waitFn not yet implemented')
         }
@@ -70,10 +76,14 @@ export default class LocalRuntime {
     }
   }
 
-  private async goto(url: string): Promise<void> {
+  private async goto(url: string, logRequests: boolean): Promise<void> {
     const {Network, Page} = this.client
     await Promise.all([Network.enable(), Page.enable()])
     await Network.setUserAgentOverride({userAgent: `Chromeless ${version}`})
+    if (logRequests) {
+      this.log(`Logging network requests`)
+      Network.requestWillBeSent(this.trafficLog.onRequest)
+    }
     await Page.navigate({url})
     await Page.loadEventFired()
     this.log(`Navigated to ${url}`)
@@ -90,6 +100,13 @@ export default class LocalRuntime {
     this.log(`Waited for ${selector}`)
   }
 
+  private async waitRequest(url: string, fn: Function): Promise<Request[]> {
+    this.log(`Waiting for request on url: ${url}`)
+    const result = await waitForRequest(this.trafficLog, url, fn, this.chromlessOptions.waitTimeout)
+    this.log(`Waited for request on url: ${url}`)
+    return result
+  }
+
   private async click(selector: string): Promise<void> {
     if (this.chromlessOptions.implicitWait) {
       this.log(`click(): Waiting for ${selector}`)
@@ -214,4 +231,4 @@ export default class LocalRuntime {
     }
   }
 
-}
+}

src/network.ts

@@ -0,0 +1,48 @@
+import { Chrome, Request, Response, RequestEvent, ResponseEvent } from './types'
+
+export default class TrafficLog {
+
+  private requests: {
+    [id: string]: Request
+  }
+
+  private responses: {
+    [id: string]: Response[]
+  }
+
+  constructor() {
+    this.requests = {}
+    this.responses = {}
+    this.onRequest = this.onRequest.bind(this)
+    this.onResponse = this.onResponse.bind(this)
+  }
+
+  private addRequest(id: string, request: Request): void {
+    this.requests[id] = request
+  }
+
+  private addResponse(id: string, response: Response): void {
+    this.responses[id].push(response)
+  }
+
+  onRequest(event: RequestEvent): void {
+      this.addRequest(event.requestId, event.request)
+  }
+
+  onResponse(event: ResponseEvent): void {
+    if (this.requests.hasOwnProperty(event.responseId)) {
+      this.addResponse(event.responseId, event.response)
+    }
+  }
+
+  async getRequests(url: string, fn: Function): Promise<any> {
+    const result = []
+    for (const id of Object.keys(this.requests)) {
+      if (this.requests[id].url.includes(url)) {
+        result.push(this.requests[id])
+      }
+    }
+    const finished = await fn(result)
+    return {requests: result, finished}
+  }
+}

src/types.ts

@@ -44,12 +44,14 @@ export type Command =
   | {
       type: 'goto'
       url: string
+      logRequests?: boolean
     }
   | {
       type: 'wait'
       timeout?: number
       selector?: string
-      fn?: string
+      fn?: Function
+      url?: string
       args?: any[]
     }
   | {
@@ -130,3 +132,52 @@ export interface CookieQuery {
   secure?: boolean
   session?: boolean
 }
+
+export interface Request {
+  url: string
+  method: string
+  headers: any
+  postData?: string
+  mixedContentType?: string
+  initialPriority: string
+  referrerPolicy: string
+  isLinkPreload?: boolean
+}
+
+export interface Response {
+  url: string
+  status: number
+  statusText: string
+  headers: any
+  headersText?: string
+  mimeType: string
+  requestHeaders?: any
+  requestHeadersText?: string
+  connectionReused: boolean
+  connectionId: number
+  fromDiskCache: boolean
+  fromServiceWorker: boolean
+  encodedDataLength: number
+  timing?: any
+  protocol?: string
+  securityState: string
+  securityDetails?: any
+}
+
+export interface RequestEvent {
+  requestId: string
+  loaderId: string
+  documentURL: string
+  request: Request
+  timestamp: number
+  initiator: any
+  redirectResponse?: Response
+}
+
+export interface ResponseEvent {
+  responseId: string,
+  loaderId: string
+  timestamp: number
+  type: string
+  response: Response
+}

src/util.ts

@@ -1,6 +1,7 @@
 import * as fs from 'fs'
 import * as path from 'path'
 import { Client, Cookie } from './types'
+import TrafficLog from './network'
 
 export const version: string = ((): string => {
   if (fs.existsSync(path.join(__dirname, '../package.json'))) {
@@ -46,6 +47,31 @@ export async function waitForNode(client: Client, selector: string, waitTimeout:
   }
 }
 
+export async function waitForRequest(trafficLog: TrafficLog, url: string, fn: Function, waitTimeout: number): Promise<Request[]> {
+  const result = await trafficLog.getRequests(url, fn)
+
+  if (!result.finished) {
+    const start = new Date().getTime()
+    return new Promise<Request[]>((resolve, reject) => {
+      const interval = setInterval(async () => {
+        if (new Date().getTime() - start > waitTimeout) {
+          clearInterval(interval)
+          reject(new Error(`waitForRequest("${url}") timed out after ${waitTimeout}ms`))
+        }
+
+        const result = await trafficLog.getRequests(url, fn)
+
+        if (result.finished) {
+          clearInterval(interval)
+          resolve(result.requests)
+        }
+      }, 500)
+    })
+  } else {
+    return result.requests
+  }
+}
+
 export async function wait(timeout: number): Promise<void> {
   return new Promise<void>((resolve, reject) => setTimeout(resolve, timeout))
 }
@@ -311,4 +337,3 @@ export function getDebugOption(): boolean {
 
   return false
 }
-