Skip to content

Commit a4b9652

Browse files
OkabintaroOkabintaro
authored
Feature: Add support for application/x-www-form-urlencoded encoding (#19)
* feat: Add http_post_form This method uses `application/x-www-form-urlencoded` instead of formatting the body as json like `http_post`. * test: Add tests for the form encoding It only tests a single value right now using httpbin but I have seen multiple work just as fine * doc: Add documentation
1 parent 0cce9a5 commit a4b9652

File tree

3 files changed

+153
-0
lines changed

3 files changed

+153
-0
lines changed

‎docs/README.md

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ LOAD http_client;
1414
### Functions
1515
- `http_get(url)`
1616
- `http_post(url, headers, params)`
17+
- Sends POST request with params encoded as a JSON object
18+
- `http_post_form(url, headers, params)`
19+
- Sends POST request with params being `application/x-www-form-urlencoded` encoded (used by many forms and some APIs)
1720

1821
### Examples
1922
#### GET
@@ -83,6 +86,44 @@ D WITH __input AS (
8386
└────────┴─────────┴─────────────┘
8487
```
8588

89+
#### POST using form encoding(application/x-www-form-urlencoded, not multipart/form-data)
90+
```sql
91+
D WITH __input AS (
92+
SELECT
93+
http_post_form(
94+
'https://httpbin.org/delay/0',
95+
headers => MAP {
96+
'accept': 'application/json',
97+
},
98+
params => MAP {
99+
'limit': 10
100+
}
101+
) AS res
102+
),
103+
__response AS (
104+
SELECT
105+
(res->>'status')::INT AS status,
106+
(res->>'reason') AS reason,
107+
unnest( from_json(((res->>'body')::JSON)->'form', '{"limit": "VARCHAR"}') ) AS features
108+
FROM
109+
__input
110+
)
111+
SELECT
112+
__response.status,
113+
__response.reason,
114+
__response.limit AS limit
115+
FROM
116+
__response
117+
;
118+
┌────────┬─────────┬─────────┐
119+
│ status │ reason │ limit │
120+
│ int32 │ varchar │ varchar │
121+
├────────┼─────────┼─────────┤
122+
│ 200 │ OK │ 10 │
123+
└────────┴─────────┴─────────┘
124+
```
125+
126+
86127
#### Full Example w/ spatial data
87128
This is the original example by @ahuarte47 inspiring this community extension.
88129

‎src/http_client_extension.cpp

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,46 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
261261
});
262262
}
263263

264+
static void HTTPPostFormRequestFunction(DataChunk &args, ExpressionState &state, Vector &result) {
265+
D_ASSERT(args.data.size() == 3);
266+
267+
using STRING_TYPE = PrimitiveType<string_t>;
268+
using LENTRY_TYPE = PrimitiveType<list_entry_t>;
269+
270+
auto &url_vector = args.data[0];
271+
auto &headers_vector = args.data[1];
272+
auto &headers_entry = ListVector::GetEntry(headers_vector);
273+
auto &body_vector = args.data[2];
274+
auto &body_entry = ListVector::GetEntry(body_vector);
275+
276+
GenericExecutor::ExecuteTernary<STRING_TYPE, LENTRY_TYPE, LENTRY_TYPE, STRING_TYPE>(
277+
url_vector, headers_vector, body_vector, result, args.size(),
278+
[&](STRING_TYPE url, LENTRY_TYPE headers, LENTRY_TYPE params) {
279+
std::string url_str = url.val.GetString();
280+
281+
// Use helper to setup client and parse URL
282+
auto client_and_path = SetupHttpClient(url_str);
283+
auto &client = client_and_path.first;
284+
auto &path = client_and_path.second;
285+
286+
// Prepare headers and parameters
287+
duckdb_httplib_openssl::Headers header_map;
288+
duckdb_httplib_openssl::Params params_map;
289+
ConvertListEntryToMap<duckdb_httplib_openssl::Headers>(headers.val, headers_entry, header_map);
290+
ConvertListEntryToMap<duckdb_httplib_openssl::Params>(params.val, body_entry, params_map);
291+
292+
// Make the POST request with headers and params
293+
auto res = client.Post(path.c_str(), header_map, params_map);
294+
if (res) {
295+
std::string response = GetJsonResponse(res->status, res->reason, res->body);
296+
return StringVector::AddString(result, response);
297+
} else {
298+
std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), "");
299+
return StringVector::AddString(result, response);
300+
}
301+
});
302+
}
303+
264304

265305
static void LoadInternal(DatabaseInstance &instance) {
266306
ScalarFunctionSet http_get("http_get");
@@ -276,6 +316,13 @@ static void LoadInternal(DatabaseInstance &instance) {
276316
{LogicalType::VARCHAR, LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), LogicalType::JSON()},
277317
LogicalType::JSON(), HTTPPostRequestFunction));
278318
ExtensionUtil::RegisterFunction(instance, http_post);
319+
320+
ScalarFunctionSet http_post_form("http_post_form");
321+
http_post_form.AddFunction(ScalarFunction(
322+
{LogicalType::VARCHAR, LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR),
323+
LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)},
324+
LogicalType::JSON(), HTTPPostFormRequestFunction));
325+
ExtensionUtil::RegisterFunction(instance, http_post_form);
279326
}
280327

281328
void HttpClientExtension::Load(DuckDB &db) {

‎test/sql/httpclient.test

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,3 +134,68 @@ FROM
134134
;
135135
----
136136
S2A_56LPN_20210930_0_L2A
137+
138+
# Confirm the POST function with form request works
139+
query III
140+
WITH __input AS (
141+
SELECT
142+
http_post_form(
143+
'https://httpbin.org/delay/0',
144+
headers => MAP {
145+
'accept': 'application/json',
146+
},
147+
params => MAP {
148+
'limit': 10
149+
}
150+
) AS res
151+
),
152+
__response AS (
153+
SELECT
154+
(res->>'status')::INT AS status,
155+
(res->>'reason') AS reason,
156+
unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features
157+
FROM
158+
__input
159+
)
160+
SELECT
161+
__response.status,
162+
__response.reason,
163+
__response.Host AS host
164+
FROM
165+
__response
166+
;
167+
----
168+
200 OK httpbin.org
169+
170+
# Confirm the POST function with form encoding transmits a single value
171+
query III
172+
WITH __input AS (
173+
SELECT
174+
http_post_form(
175+
'https://httpbin.org/delay/0',
176+
headers => MAP {
177+
'accept': 'application/json',
178+
},
179+
params => MAP {
180+
'limit': 10
181+
}
182+
) AS res
183+
),
184+
__response AS (
185+
SELECT
186+
(res->>'status')::INT AS status,
187+
(res->>'reason') AS reason,
188+
unnest( from_json(((res->>'body')::JSON)->'form', '{"limit": "VARCHAR"}') ) AS features
189+
FROM
190+
__input
191+
)
192+
SELECT
193+
__response.status,
194+
__response.reason,
195+
__response.limit AS limit
196+
FROM
197+
__response
198+
;
199+
----
200+
200 OK 10
201+

0 commit comments

Comments
 (0)