Try harder to preserve raw header formatting with passthrough transforms

pimterry · pimterry · commit bc10695104c3 · 2025-04-11T15:39:02.000+02:00
We now preserve more-or-less everything for every header that you don't
directly modify. This should affect header dupe handling, header key
casing and maybe ordering, but nothing else, so shouldn't cause any
meaningful effects in well-behaved scenarios.
diff --git a/src/rules/passthrough-handling.ts b/src/rules/passthrough-handling.ts
@@ -6,13 +6,15 @@ import { oneLine } from 'common-tags';
 import CacheableLookup from 'cacheable-lookup';
 import * as semver from 'semver';
 
-import { CompletedBody, Headers } from '../types';
+import { CompletedBody, Headers, RawHeaders } from '../types';
 import { byteLength } from '../util/util';
 import { asBuffer } from '../util/buffer-utils';
 import { isLocalhostAddress, normalizeIP } from '../util/socket-util';
 import { CachedDns, dnsLookup, DnsLookupFunction } from '../util/dns';
 import { isMockttpBody, encodeBodyBuffer } from '../util/request-utils';
 import { areFFDHECurvesSupported } from '../util/openssl-compat';
+import { ErrorLike } from '../util/error';
+import { getHeaderValue } from '../util/header-utils';
 
 import {
     CallbackRequestResult,
@@ -23,7 +25,6 @@ import {
     CADefinition,
     PassThroughLookupOptions
 } from './passthrough-handling-definitions';
-import { ErrorLike } from '../util/error';
 
 // TLS settings for proxied connections, intended to avoid TLS fingerprint blocking
 // issues so far as possible, by closely emulating a Firefox Client Hello:
@@ -266,18 +267,18 @@ export function getH2HeadersAfterModification(
 // Helper to handle content-length nicely for you when rewriting requests with callbacks
 export function getContentLengthAfterModification(
     body: string | Uint8Array | Buffer,
-    originalHeaders: Headers,
-    replacementHeaders: Headers | undefined,
+    originalHeaders: Headers | RawHeaders,
+    replacementHeaders: Headers | RawHeaders | undefined,
     mismatchAllowed: boolean = false
 ): string | undefined {
     // If there was a content-length header, it might now be wrong, and it's annoying
     // to need to set your own content-length override when you just want to change
     // the body. To help out, if you override the body but don't explicitly override
     // the (now invalid) content-length, then we fix it for you.
 
-    if (!_.has(originalHeaders, 'content-length')) {
+    if (getHeaderValue(originalHeaders, 'content-length') === undefined) {
         // Nothing to override - use the replacement value, or undefined
-        return (replacementHeaders || {})['content-length'];
+        return getHeaderValue(replacementHeaders || {}, 'content-length');
     }
 
     if (!replacementHeaders) {
@@ -288,14 +289,14 @@ export function getContentLengthAfterModification(
     }
 
     // There was a content length before, and you're replacing the headers entirely
-    const lengthOverride = replacementHeaders['content-length']?.toString();
+    const lengthOverride = getHeaderValue(replacementHeaders, 'content-length')?.toString();
 
     // If you're setting the content-length to the same as the origin headers, even
     // though that's the wrong value, it *might* be that you're just extending the
     // existing headers, and you're doing this by accident (we can't tell for sure).
     // We use invalid content-length as instructed, but print a warning just in case.
     if (
-        lengthOverride === originalHeaders['content-length'] &&
+        lengthOverride === getHeaderValue(originalHeaders, 'content-length') &&
         lengthOverride !== byteLength(body).toString() &&
         !mismatchAllowed // Set for HEAD responses
     ) {
diff --git a/src/rules/requests/request-handlers.ts b/src/rules/requests/request-handlers.ts
@@ -40,7 +40,9 @@ import {
     pairFlatRawHeaders,
     findRawHeaderIndex,
     dropDefaultHeaders,
-    validateHeader
+    validateHeader,
+    updateRawHeaders,
+    getHeaderValue
 } from '../../util/header-utils';
 import { streamToBuffer, asBuffer } from '../../util/buffer-utils';
 import {
@@ -484,8 +486,6 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
         let headersManuallyModified = false;
 
         if (this.transformRequest) {
-            let headers = rawHeadersToObject(rawHeaders);
-
             const {
                 replaceMethod,
                 updateHeaders,
@@ -502,14 +502,9 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
             }
 
             if (updateHeaders) {
-                headers = {
-                    ...headers,
-                    ...updateHeaders
-                };
-                headersManuallyModified = true;
+                rawHeaders = updateRawHeaders(rawHeaders, updateHeaders);
             } else if (replaceHeaders) {
-                headers = { ...replaceHeaders };
-                headersManuallyModified = true;
+                rawHeaders = objectHeadersToRaw(replaceHeaders);
             }
 
             if (replaceBody) {
@@ -564,22 +559,22 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
                 // We always re-encode the body to match the resulting content-encoding header:
                 reqBodyOverride = await encodeBodyBuffer(
                     reqBodyOverride,
-                    headers
+                    rawHeaders
                 );
 
-                headers['content-length'] = getContentLengthAfterModification(
+                const updatedCLHeader = getContentLengthAfterModification(
                     reqBodyOverride,
                     clientReq.headers,
-                    (updateHeaders && updateHeaders['content-length'] !== undefined)
-                        ? headers // Iff you replaced the content length
-                        : replaceHeaders,
+                    (updateHeaders && getHeaderValue(updateHeaders, 'content-length') !== undefined)
+                        ? rawHeaders // Iff you replaced the content length
+                        : replaceHeaders
                 );
-            }
 
-            if (headersManuallyModified || reqBodyOverride) {
-                // If the headers have been updated (implicitly or explicitly) we need to regenerate them. We avoid
-                // this if possible, because it normalizes headers, which is slightly lossy (e.g. they're lowercased).
-                rawHeaders = objectHeadersToRaw(headers);
+                if (updatedCLHeader !== undefined) {
+                    rawHeaders = updateRawHeaders(rawHeaders, {
+                        'content-length': updatedCLHeader
+                    });
+                }
             }
         } else if (this.beforeRequest) {
             const clientRawHeaders = rawHeaders;
@@ -827,9 +822,6 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
                 }
 
                 if (this.transformResponse) {
-                    let responseHeadersModified = false;
-                    let serverHeaders = rawHeadersToObject(serverRawHeaders);
-
                     const {
                         replaceStatus,
                         updateHeaders,
@@ -847,14 +839,9 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
                     }
 
                     if (updateHeaders) {
-                        serverHeaders = {
-                            ...serverHeaders,
-                            ...updateHeaders
-                        };
-                        responseHeadersModified = true;
+                        serverRawHeaders = updateRawHeaders(serverRawHeaders, updateHeaders);
                     } else if (replaceHeaders) {
-                        serverHeaders = { ...replaceHeaders };
-                        responseHeadersModified = true;
+                        serverRawHeaders = objectHeadersToRaw(replaceHeaders);
                     }
 
                     if (replaceBody) {
@@ -915,24 +902,23 @@ export class PassThroughHandler extends PassThroughHandlerDefinition {
                         // so we re-encode the body to match the resulting content-encoding header:
                         resBodyOverride = await encodeBodyBuffer(
                             resBodyOverride,
-                            serverHeaders
+                            serverRawHeaders
                         );
 
-                        serverHeaders['content-length'] = getContentLengthAfterModification(
+                        const updatedCLHeader = getContentLengthAfterModification(
                             resBodyOverride,
                             serverRes.headers,
-                            (updateHeaders && updateHeaders['content-length'] !== undefined)
-                                ? serverHeaders // Iff you replaced the content length
+                            (updateHeaders && getHeaderValue(updateHeaders, 'content-length') !== undefined)
+                                ? serverRawHeaders // Iff you replaced the content length
                                 : replaceHeaders,
                             method === 'HEAD' // HEAD responses are allowed mismatched content-length
                         );
-                        responseHeadersModified = true;
-                    }
 
-                    if (responseHeadersModified) {
-                        // If the headers have been updated (implicitly or explicitly) we need to regenerate them. We avoid
-                        // this if possible, because it normalizes headers, which is slightly lossy (e.g. they're lowercased).
-                        serverRawHeaders = objectHeadersToRaw(serverHeaders);
+                        if (updatedCLHeader !== undefined) {
+                            serverRawHeaders = updateRawHeaders(serverRawHeaders, {
+                                'content-length': updatedCLHeader
+                            });
+                        }
                     }
                 } else if (this.beforeResponse) {
                     let modifiedRes: CallbackResponseResult | void;
diff --git a/src/util/header-utils.ts b/src/util/header-utils.ts
@@ -1,3 +1,4 @@
+import * as _ from 'lodash';
 import * as http from 'http';
 
 import {
@@ -27,6 +28,19 @@ with. Those are:
 export const findRawHeader = (rawHeaders: RawHeaders, targetKey: string) =>
     rawHeaders.find(([key]) => key.toLowerCase() === targetKey);
 
+export const getHeaderValue = (headers: Headers | RawHeaders, targetKey: Lowercase<string>) => {
+    if (Array.isArray(headers)) {
+        return findRawHeader(headers, targetKey)?.[1];
+    } else {
+        const value = headers[targetKey];
+        if (Array.isArray(value)) {
+            return value[0];
+        } else {
+            return value;
+        }
+    }
+};
+
 export const findRawHeaderIndex = (rawHeaders: RawHeaders, targetKey: string) =>
     rawHeaders.findIndex(([key]) => key.toLowerCase() === targetKey);
 
@@ -158,6 +172,50 @@ export function objectHeadersToFlat(headers: Headers): string[] {
     return flatHeaders;
 }
 
+/**
+ * Combine the given headers with the raw headers, preserving the raw header details where
+ * possible. Headers keys that exist in the raw headers (case insensitive) will be overridden,
+ * while undefined header values will remove the header from the raw headers entirely.
+ *
+ * When proxying we often have raw received headers that we want to forward upstream exactly
+ * as they were received, but we also want to add or modify a subset of those headers. This
+ * method carefully does that - preserving everything that isn't actively modified as-is.
+ */
+export function updateRawHeaders(
+    rawHeaders: RawHeaders,
+    headers: Headers
+) {
+    const updatedRawHeaders = [...rawHeaders];
+    const rawHeaderKeys = updatedRawHeaders.map(([key]) => key.toLowerCase());
+
+    for (const key of Object.keys(headers)) {
+        const lowerCaseKey = key.toLowerCase();
+        for (let i = 0; i < rawHeaderKeys.length; i++) {
+            // If you insert a header that already existed, remove all previous values
+            if (rawHeaderKeys[i] === lowerCaseKey) {
+                updatedRawHeaders.splice(i, 1);
+                rawHeaderKeys.splice(i, 1);
+            }
+        }
+    }
+
+    // We do all removals in advance, then do all additions here, to ensure that adding
+    // a new header twice works correctly.
+
+    for (const [key, value] of Object.entries(headers)) {
+        // Skip (effectively delete) undefined/null values from the headers
+        if (value === undefined || value === null) continue;
+
+        if (Array.isArray(value)) {
+            value.forEach((v) => updatedRawHeaders.push([key, v]));
+        } else {
+            updatedRawHeaders.push([key, value]);
+        }
+    }
+
+    return updatedRawHeaders;
+}
+
 // See https://httptoolkit.com/blog/translating-http-2-into-http-1/ for details on the
 // transformations required between H2 & H1 when proxying.
 export function h2HeadersToH1(h2Headers: RawHeaders): RawHeaders {
@@ -217,4 +275,33 @@ export function validateHeader(name: string, value: string | string[]): boolean
     } catch (e) {
         return false;
     }
+}
+
+/**
+ * Set the value of a given header, overwriting it if present or otherwise adding it as a new header.
+ *
+ * For header objects, this overwrites all values. For raw headers, this overwrites the last value, so
+ * if multiple values are present others may remain. In general you probably don't want to use this
+ * for headers that could legally have multiple values present.
+ */
+export const setHeaderValue = (
+    headers: Headers | RawHeaders,
+    headerKey: string,
+    headerValue: string,
+    options: { prepend?: true } = {}
+) => {
+    const lowercaseHeaderKey = headerKey.toLowerCase();
+
+    if (Array.isArray(headers)) {
+        const headerPair = _.findLast(headers, ([key]) => key.toLowerCase() === lowercaseHeaderKey);
+        if (headerPair) {
+            headerPair[1] = headerValue;
+        } else {
+            if (options.prepend) headers.unshift([headerKey, headerValue]);
+            else headers.push([headerKey, headerValue]);
+        }
+    } else {
+        const existingKey = Object.keys(headers).find(k => k.toLowerCase() === lowercaseHeaderKey);
+        headers[existingKey || headerKey] = headerValue;
+    }
 }
diff --git a/src/util/request-utils.ts b/src/util/request-utils.ts
@@ -33,6 +33,7 @@ import {
 } from './buffer-utils';
 import {
     flattenPairedRawHeaders,
+    getHeaderValue,
     objectHeadersToFlat,
     objectHeadersToRaw,
     pairFlatRawHeaders,
@@ -89,8 +90,8 @@ export function isHttp2(
         ('stream' in message && 'createPushResponse' in message); // H2 response
 }
 
-export async function encodeBodyBuffer(buffer: Uint8Array, headers: Headers) {
-    const contentEncoding = headers['content-encoding'];
+export async function encodeBodyBuffer(buffer: Uint8Array, headers: Headers | RawHeaders) {
+    const contentEncoding = getHeaderValue(headers, 'content-encoding');
 
     // We skip encodeBuffer entirely if possible - this isn't strictly necessary, but it's useful
     // so you can drop the http-encoding package in bundling downstream without issue in cases
diff --git a/test/header-utils.spec.ts b/test/header-utils.spec.ts
diff --git a/test/integration/proxying/proxy-transforms.spec.ts b/test/integration/proxying/proxy-transforms.spec.ts
