Added response headers to fetch API (#8486)

* Added response headers to fetch with flag, and headersreceived callback

* Added test for headers received

* onreadystatechange callback, removed define and responseheaders from fetch struct

* added get_response_headers utility functions

* added header unpacking functions

* Fixes and cleanup

* Fixed tests, fixed header helpers, fixed JS interop

* fixed indentation in Fetch.js

* Altered comment for get_response_headers_length

* Changed ready state comment

* Changed internal fetch_get_response_headers names, added arg names

* Additional header file comments

* Correct mapping of function names

Author: coopersimon

src/Fetch.js

@@ -276,7 +276,7 @@ function __emscripten_fetch_cache_data(db, fetch, data, onsuccess, onerror) {
 }
 #endif // ~FETCH_SUPPORT_INDEXEDDB
 
-function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) {
+function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
   var url = HEAPU32[fetch + {{{ C_STRUCTS.emscripten_fetch_t.url }}} >> 2];
   if (!url) {
 #if FETCH_DEBUG
@@ -438,6 +438,13 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) {
     if (xhr.statusText) stringToUTF8(xhr.statusText, fetch + {{{ C_STRUCTS.emscripten_fetch_t.statusText }}}, 64);
     if (onprogress) onprogress(fetch, xhr, e);
   }
+  xhr.onreadystatechange = function(e) {
+    HEAPU16[fetch + {{{ C_STRUCTS.emscripten_fetch_t.readyState }}} >> 1] = xhr.readyState;
+    if (xhr.readyState >= 2) {
+      HEAPU16[fetch + {{{ C_STRUCTS.emscripten_fetch_t.status }}} >> 1] = xhr.status;
+    }
+    if (onreadystatechange) onreadystatechange(fetch, xhr, e);
+  }
 #if FETCH_DEBUG
   console.log('fetch: xhr.send(data=' + data + ')');
 #endif
@@ -451,14 +458,15 @@ function __emscripten_fetch_xhr(fetch, onsuccess, onerror, onprogress) {
   }
 }
 
-function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) {
+function emscripten_start_fetch(fetch, successcb, errorcb, progresscb, readystatechangecb) {
   if (typeof Module !== 'undefined') Module['noExitRuntime'] = true; // If we are the main Emscripten runtime, we should not be closing down.
 
   var fetch_attr = fetch + {{{ C_STRUCTS.emscripten_fetch_t.__attributes }}};
   var requestMethod = UTF8ToString(fetch_attr);
   var onsuccess = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.onsuccess }}} >> 2];
   var onerror = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.onerror }}} >> 2];
   var onprogress = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.onprogress }}} >> 2];
+  var onreadystatechange = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.onreadystatechange }}} >> 2];
   var fetchAttributes = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.attributes }}} >> 2];
   var fetchAttrLoadToMemory = !!(fetchAttributes & {{{ cDefine('EMSCRIPTEN_FETCH_LOAD_TO_MEMORY') }}});
   var fetchAttrStreamData = !!(fetchAttributes & {{{ cDefine('EMSCRIPTEN_FETCH_STREAM_DATA') }}});
@@ -490,11 +498,19 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) {
     else if (errorcb) errorcb(fetch);
   };
 
+  var reportReadyStateChange = function(fetch, xhr, e) {
+#if FETCH_DEBUG
+    console.log('fetch: ready state change. e: ' + e);
+#endif
+    if (onreadystatechange) {{{ makeDynCall('vi') }}}(onreadystatechange, fetch);
+    else if (readystatechangecb) readystatechangecb(fetch);
+  }
+
   var performUncachedXhr = function(fetch, xhr, e) {
 #if FETCH_DEBUG
     console.error('fetch: starting (uncached) XHR: ' + e);
 #endif
-    __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress);
+    __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress, reportReadyStateChange);
   };
 
 #if FETCH_SUPPORT_INDEXEDDB
@@ -523,7 +539,7 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) {
 #if FETCH_DEBUG
     console.error('fetch: starting (cached) XHR: ' + e);
 #endif
-    __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress);
+    __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress, reportReadyStateChange);
   };
 
   // Should we try IndexedDB first?
@@ -545,7 +561,7 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) {
   } else if (!fetchAttrReplace) {
     __emscripten_fetch_load_cached_data(Fetch.dbInstance, fetch, reportSuccess, fetchAttrNoDownload ? reportError : (fetchAttrPersistFile ? performCachedXhr : performUncachedXhr));
   } else if (!fetchAttrNoDownload) {
-    __emscripten_fetch_xhr(fetch, fetchAttrPersistFile ? cacheResultAndReportSuccess : reportSuccess, reportError, reportProgress);
+    __emscripten_fetch_xhr(fetch, fetchAttrPersistFile ? cacheResultAndReportSuccess : reportSuccess, reportError, reportProgress, reportReadyStateChange);
   } else {
 #if FETCH_DEBUG
     console.error('fetch: Invalid combination of flags passed.');
@@ -554,7 +570,18 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb) {
   }
   return fetch;
 #else // !FETCH_SUPPORT_INDEXEDDB
-  __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress);
+  __emscripten_fetch_xhr(fetch, reportSuccess, reportError, reportProgress, reportReadyStateChange);
   return fetch;
 #endif // ~FETCH_SUPPORT_INDEXEDDB
 }
+
+function _fetch_get_response_headers_length(id) {
+    return lengthBytesUTF8(Fetch.xhrs[id-1].getAllResponseHeaders()) + 1;
+}
+
+function _fetch_get_response_headers(id, dst, dstSizeBytes) {
+    var responseHeaders = Fetch.xhrs[id-1].getAllResponseHeaders();
+    var lengthBytes = lengthBytesUTF8(responseHeaders) + 1;
+    stringToUTF8(responseHeaders, dst, dstSizeBytes);
+    return Math.min(lengthBytes, dstSizeBytes);
+}

src/library_fetch.js

@@ -18,6 +18,8 @@ var LibraryFetch = {
   _emscripten_get_fetch_work_queue: function() {
     return _fetch_work_queue;
   },
+  _emscripten_fetch_get_response_headers_length: _fetch_get_response_headers_length,
+  _emscripten_fetch_get_response_headers: _fetch_get_response_headers,
 
 #if FETCH_SUPPORT_INDEXEDDB
   $__emscripten_fetch_delete_cached_data: __emscripten_fetch_delete_cached_data,

src/struct_info.json

@@ -1566,6 +1566,7 @@
                 "onsuccess",
                 "onerror",
                 "onprogress",
+                "onreadystatechange",
                 "attributes",
                 "timeoutMSecs",
                 "withCredentials",

system/include/emscripten/fetch.h

@@ -68,6 +68,7 @@ typedef struct emscripten_fetch_attr_t
 	void (*onsuccess)(struct emscripten_fetch_t *fetch);
 	void (*onerror)(struct emscripten_fetch_t *fetch);
 	void (*onprogress)(struct emscripten_fetch_t *fetch);
+	void (*onreadystatechange)(struct emscripten_fetch_t *fetch);
 
 	// EMSCRIPTEN_FETCH_* attributes
 	uint32_t attributes;
@@ -185,6 +186,24 @@ EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeou
 // onerror() handler will be called in the calling thread before this function returns.
 EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch);
 
+// Gets the size (in bytes) of the response headers as plain text.
+// This must be called on the same thread as the fetch originated on.
+// Note that this will return 0 if readyState < HEADERS_RECEIVED.
+size_t emscripten_fetch_get_response_headers_length(emscripten_fetch_t *fetch);
+
+// Gets the response headers as plain text. dstSizeBytes should be headers_length + 1 (for the null terminator).
+// This must be called on the same thread as the fetch originated on.
+size_t emscripten_fetch_get_response_headers(emscripten_fetch_t *fetch, char *dst, size_t dstSizeBytes);
+
+// Converts the plain text headers into an array of strings. This array takes the form
+// {"key1", "value1", "key2", "value2", "key3", "value3", ..., 0 }; Note especially that the array
+// is terminated with a null pointer.
+char **emscripten_fetch_unpack_response_headers(const char *headersString);
+
+// This frees the memory used by the array of headers. Call this when finished with the data returned
+// by emscripten_fetch_unpack_response_headers.
+void emscripten_fetch_free_unpacked_response_headers(char **unpackedHeaders);
+
 #define emscripten_asmfs_open_t int
 
 // The following flags specify how opening files for reading works (from strictest behavior to most flexible)

system/lib/fetch/emscripten_fetch.cpp

@@ -40,6 +40,9 @@ __emscripten_fetch_queue* _emscripten_get_fetch_queue() {
   }
   return queue;
 }
+
+int32_t _emscripten_fetch_get_response_headers_length(int32_t fetchID);
+int32_t _emscripten_fetch_get_response_headers(int32_t fetchID, int32_t dst, int32_t dstSizeBytes);
 }
 
 void emscripten_proxy_fetch(emscripten_fetch_t* fetch) {
@@ -100,6 +103,7 @@ emscripten_fetch_t* emscripten_fetch(emscripten_fetch_attr_t* fetch_attr, const
   fetch->__attributes.onerror = fetch_attr->onerror;
   fetch->__attributes.onsuccess = fetch_attr->onsuccess;
   fetch->__attributes.onprogress = fetch_attr->onprogress;
+  fetch->__attributes.onreadystatechange = fetch_attr->onreadystatechange;
 #define STRDUP_OR_ABORT(s, str_to_dup)                                                             \
   if (str_to_dup) {                                                                                \
     s = strdup(str_to_dup);                                                                        \
@@ -239,6 +243,63 @@ EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t* fetch) {
   return EMSCRIPTEN_RESULT_SUCCESS;
 }
 
+size_t emscripten_fetch_get_response_headers_length(emscripten_fetch_t *fetch) {
+  if (!fetch || fetch->readyState < 2) return 0;
+
+  return (size_t)_emscripten_fetch_get_response_headers_length((int32_t)fetch->id);
+}
+
+size_t emscripten_fetch_get_response_headers(emscripten_fetch_t *fetch, char *dst, size_t dstSizeBytes) {
+  if (!fetch || fetch->readyState < 2) return 0;
+
+  return (size_t)_emscripten_fetch_get_response_headers((int32_t)fetch->id, (int32_t)dst, (int32_t)dstSizeBytes);
+}
+
+char **emscripten_fetch_unpack_response_headers(const char *headersString) {
+  // Get size of output array and allocate.
+  size_t numHeaders = 0;
+  for(const char *pos = strchr(headersString, '\n'); pos; pos = strchr(pos + 1, '\n'))
+  {
+    numHeaders++;
+  }
+  char **unpackedHeaders = (char**)malloc(sizeof(char*) * ((numHeaders * 2) + 1));
+  unpackedHeaders[numHeaders * 2] = NULL;
+
+  // Allocate each header.
+  const char *rowStart = headersString;
+  const char *rowEnd = strchr(rowStart, '\n');
+  for(size_t headerNum = 0; rowEnd; headerNum += 2)
+  {
+    const char *split = strchr(rowStart, ':');
+    size_t headerSize = (size_t)split - (size_t)rowStart;
+    char* header = (char*)malloc(headerSize + 1);
+    strncpy(header, rowStart, headerSize);
+    header[headerSize] = '\0';
+
+    size_t valueSize = (size_t)rowEnd - (size_t)split;
+    char* value = (char*)malloc(valueSize + 1);
+    strncpy(value, split + 1, valueSize);
+    value[valueSize] = '\0';
+
+    unpackedHeaders[headerNum] = header;
+    unpackedHeaders[headerNum+1] = value;
+
+    rowStart = rowEnd + 1;
+    rowEnd = strchr(rowStart, '\n');
+  }
+
+  return unpackedHeaders;
+}
+
+void emscripten_fetch_free_unpacked_response_headers(char **unpackedHeaders) {
+  if(unpackedHeaders)
+  {
+    for(size_t i = 0; unpackedHeaders[i]; ++i)
+      free((void*)unpackedHeaders[i]);
+    free((void*)unpackedHeaders);
+  }
+}
+
 static void fetch_free(emscripten_fetch_t* fetch) {
   fetch->id = 0;
   free((void*)fetch->data);

tests/fetch/headers_received.cpp

@@ -0,0 +1,61 @@
+// Copyright 2016 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <emscripten/fetch.h>
+
+void readyStateChange(emscripten_fetch_t *fetch)
+{
+  if(fetch->readyState != 2) return;
+
+  size_t headersLengthBytes = emscripten_fetch_get_response_headers_length(fetch) + 1;
+  char *headerString = new char[headersLengthBytes];
+
+  assert(headerString);
+  emscripten_fetch_get_response_headers(fetch, headerString, headersLengthBytes);
+  printf("Got headers: %s\n", headerString);
+
+  char **responseHeaders = emscripten_fetch_unpack_response_headers(headerString);
+  assert(responseHeaders);
+
+  delete[] headerString;
+
+  int numHeaders = 0;
+  for(; responseHeaders[numHeaders * 2]; ++numHeaders)
+  {
+    // Check both the header and its value are present.
+    assert(responseHeaders[(numHeaders * 2) + 1]);
+    printf("Got response header: %s:%s\n", responseHeaders[numHeaders * 2], responseHeaders[(numHeaders * 2) + 1]);
+  }
+
+  printf("Finished receiving %d headers from URL %s.\n", numHeaders, fetch->url);
+
+  emscripten_fetch_free_unpacked_response_headers(responseHeaders);
+
+#ifdef REPORT_RESULT
+  REPORT_RESULT(0);
+#endif
+}
+
+void success(emscripten_fetch_t *fetch)
+{
+  printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url);
+  // The data is now available at fetch->data[0] through fetch->data[fetch->numBytes-1];
+  emscripten_fetch_close(fetch); // Free data associated with the fetch.
+}
+
+int main()
+{
+  emscripten_fetch_attr_t attr;
+  emscripten_fetch_attr_init(&attr);
+  strcpy(attr.requestMethod, "GET");
+  attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_REPLACE;
+  attr.onsuccess = success;
+  attr.onreadystatechange = readyStateChange;
+  attr.timeoutMSecs = 2*60;
+  emscripten_fetch(&attr, "myfile.dat");
+}