Skip to content

Commit 83c81e8

Browse files
leonardovillelaleonardovillela
committed
feat: add new option fileWriteStreamHandler
1 parent c21ef44 commit 83c81e8

8 files changed

+288
-59
lines changed

‎README.md

+11-3
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ _**Note: v2 is coming soon!**_
6565

6666
- [Fast (~900-2500 mb/sec)](#benchmarks) & streaming multipart parser
6767
- Automatically writing file uploads to disk (optional, see
68-
[`options.storeFiles`](#options))
68+
[`options.fileWriteStreamHandler`](#options))
6969
- [Plugins API](#useplugin-plugin) - allowing custom parsers and plugins
7070
- Low memory footprint
7171
- Graceful error handling
@@ -335,8 +335,16 @@ See it's defaults in [src/Formidable.js DEFAULT_OPTIONS](./src/Formidable.js)
335335
for incoming files, set this to some hash algorithm, see
336336
[crypto.createHash](https://nodejs.org/api/crypto.html#crypto_crypto_createhash_algorithm_options)
337337
for available algorithms
338-
- `options.storeFiles` **{boolean}** - default `true`; to store uploaded file(s)
339-
in `uploadDir` on host machine or not and only parse the file(s).
338+
- `options.fileWriteStreamHandler` **{function}** - default `null`, which by
339+
default writes to host machine file system every file parsed; The function
340+
should return an instance of a
341+
[Writable stream](https://nodejs.org/api/stream.html#stream_class_stream_writable)
342+
that will receive the uploaded file data. With this option, you can have any
343+
custom behavior regarding where the uploaded file data will be streamed for.
344+
If you are looking to write the file uploaded in other types of cloud storages
345+
(AWS S3, Azure blob storage, Google cloud storage) or private file storage,
346+
this is the option you're looking for. When this option is defined the default
347+
behavior of writing the file in the host machine file system is lost.
340348
- `options.multiples` **{boolean}** - default `false`; when you call the
341349
`.parse` method, the `files` argument (of the callback) will contain arrays of
342350
files for inputs which submit multiple files using the HTML5 `multiple`

‎examples/not-store-files.js ‎examples/log-file-content-to-console.js

+16-5
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,27 @@
11
'use strict';
22

33
const http = require('http');
4+
const { Writable } = require('stream');
45
const formidable = require('../src/index');
56

67
const server = http.createServer((req, res) => {
78
if (req.url === '/api/upload' && req.method.toLowerCase() === 'post') {
89
// parse a file upload
9-
const form = formidable({ multiples: true, storeFiles: false });
10+
const form = formidable({
11+
fileWriteStreamHandler: () => {
12+
const writable = Writable();
13+
// eslint-disable-next-line no-underscore-dangle
14+
writable._write = (chunk, enc, next) => {
15+
console.log(chunk.toString());
16+
next();
17+
};
18+
return writable;
19+
},
20+
});
1021

11-
form.parse(req, (err, fields, files) => {
12-
res.writeHead(200, { 'Content-Type': 'application/json' });
13-
res.end(JSON.stringify({ fields, files }, null, 2));
22+
form.parse(req, () => {
23+
res.writeHead(200);
24+
res.end();
1425
});
1526

1627
return;
@@ -22,7 +33,7 @@ const server = http.createServer((req, res) => {
2233
<h2>With Node.js <code>"http"</code> module</h2>
2334
<form action="/api/upload" enctype="multipart/form-data" method="post">
2435
<div>Text field title: <input type="text" name="title" /></div>
25-
<div>File: <input type="file" name="multipleFiles" multiple="multiple" /></div>
36+
<div>File: <input type="file" name="file" /></div>
2637
<input type="submit" value="Upload" />
2738
</form>
2839
`);

‎examples/store-files-on-s3.js

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// To test this example you have to install aws-sdk nodejs package and create a bucket named "demo-bucket"
2+
3+
'use strict';
4+
5+
const http = require('http');
6+
const { PassThrough } = require('stream');
7+
// eslint-disable-next-line import/no-unresolved
8+
const AWS = require('aws-sdk');
9+
const formidable = require('../src/index');
10+
11+
const s3Client = new AWS.S3({
12+
credentials: {
13+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
14+
secretAccessKey: process.env.AWS_SECRET_KEY,
15+
},
16+
});
17+
18+
const uploadStream = (filename) => {
19+
const pass = PassThrough();
20+
s3Client.upload(
21+
{
22+
Bucket: 'demo-bucket',
23+
Key: filename,
24+
Body: pass,
25+
},
26+
(err, data) => {
27+
console.log(err, data);
28+
},
29+
);
30+
31+
return pass;
32+
};
33+
34+
const server = http.createServer((req, res) => {
35+
if (req.url === '/api/upload' && req.method.toLowerCase() === 'post') {
36+
// parse a file upload
37+
const form = formidable({
38+
fileWriteStreamHandler: uploadStream,
39+
});
40+
41+
form.parse(req, () => {
42+
res.writeHead(200);
43+
res.end();
44+
});
45+
46+
return;
47+
}
48+
49+
// show a file upload form
50+
res.writeHead(200, { 'Content-Type': 'text/html' });
51+
res.end(`
52+
<h2>With Node.js <code>"http"</code> module</h2>
53+
<form action="/api/upload" enctype="multipart/form-data" method="post">
54+
<div>Text field title: <input type="text" name="title" /></div>
55+
<div>File: <input type="file" name="file"/></div>
56+
<input type="submit" value="Upload" />
57+
</form>
58+
`);
59+
});
60+
61+
server.listen(3000, () => {
62+
console.log('Server listening on http://localhost:3000 ...');
63+
});

‎src/Formidable.js

+6-5
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ const DEFAULT_OPTIONS = {
2424
encoding: 'utf-8',
2525
hash: false,
2626
uploadDir: os.tmpdir(),
27-
storeFiles: true,
2827
multiples: false,
2928
enabledPlugins: ['octetstream', 'querystring', 'multipart', 'json'],
29+
fileWriteStreamHandler: null,
3030
};
3131

3232
const PersistentFile = require('./PersistentFile');
@@ -445,14 +445,15 @@ class IncomingForm extends EventEmitter {
445445
}
446446

447447
_newFile({ path: filePath, filename: name, mime: type }) {
448-
return this.options.storeFiles
449-
? new PersistentFile({
450-
path: filePath,
448+
return this.options.fileWriteStreamHandler
449+
? new VolatileFile({
451450
name,
452451
type,
452+
createFileWriteStream: this.options.fileWriteStreamHandler,
453453
hash: this.options.hash,
454454
})
455-
: new VolatileFile({
455+
: new PersistentFile({
456+
path: filePath,
456457
name,
457458
type,
458459
hash: this.options.hash,

‎src/VolatileFile.js

+25-9
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ class VolatileFile extends EventEmitter {
1414
this.type = null;
1515
this.hash = null;
1616

17+
this._writeStream = null;
18+
1719
// eslint-disable-next-line guard-for-in, no-restricted-syntax
1820
for (const key in properties) {
1921
this[key] = properties[key];
@@ -26,11 +28,16 @@ class VolatileFile extends EventEmitter {
2628
}
2729
}
2830

29-
// eslint-disable-next-line class-methods-use-this
30-
open() {}
31+
open() {
32+
this._writeStream = this.createFileWriteStream(this.name);
33+
this._writeStream.on('error', (err) => {
34+
this.emit('error', err);
35+
});
36+
}
3137

32-
// eslint-disable-next-line class-methods-use-this
33-
destroy() {}
38+
destroy() {
39+
this._writeStream.destroy();
40+
}
3441

3542
toJSON() {
3643
const json = {
@@ -56,17 +63,26 @@ class VolatileFile extends EventEmitter {
5663
this.hash.update(buffer);
5764
}
5865

59-
this.size += buffer.length;
60-
this.emit('progress', this.size);
61-
cb();
66+
if (this._writeStream.closed || this._writeStream.destroyed) {
67+
cb();
68+
return;
69+
}
70+
71+
this._writeStream.write(buffer, () => {
72+
this.size += buffer.length;
73+
this.emit('progress', this.size);
74+
cb();
75+
});
6276
}
6377

6478
end(cb) {
6579
if (this.hash) {
6680
this.hash = this.hash.digest('hex');
6781
}
68-
this.emit('end');
69-
cb();
82+
this._writeStream.end(() => {
83+
this.emit('end');
84+
cb();
85+
});
7086
}
7187
}
7288

‎test/integration/test-file-write-stream-handler-option.js

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
'use strict';
2+
3+
const fs = require('fs');
4+
const os = require('os');
5+
const http = require('http');
6+
const path = require('path');
7+
const assert = require('assert');
8+
9+
const formidable = require('../../src/index');
10+
11+
const PORT = 13532;
12+
const DEFAULT_UPLOAD_DIR = path.join(
13+
os.tmpdir(),
14+
'test-store-files-option-default',
15+
);
16+
const CUSTOM_UPLOAD_DIR = path.join(
17+
os.tmpdir(),
18+
'test-store-files-option-custom',
19+
);
20+
const CUSTOM_UPLOAD_FILE_PATH = path.join(CUSTOM_UPLOAD_DIR, 'test-file');
21+
const testFilePath = path.join(
22+
path.dirname(__dirname),
23+
'fixture',
24+
'file',
25+
'binaryfile.tar.gz',
26+
);
27+
28+
const server = http.createServer((req, res) => {
29+
if (!fs.existsSync(DEFAULT_UPLOAD_DIR)) {
30+
fs.mkdirSync(DEFAULT_UPLOAD_DIR);
31+
}
32+
if (!fs.existsSync(CUSTOM_UPLOAD_DIR)) {
33+
fs.mkdirSync(CUSTOM_UPLOAD_DIR);
34+
}
35+
const form = formidable({
36+
uploadDir: DEFAULT_UPLOAD_DIR,
37+
fileWriteStreamHandler: () => fs.createWriteStream(CUSTOM_UPLOAD_FILE_PATH),
38+
});
39+
40+
form.parse(req, (err, fields, files) => {
41+
assert.strictEqual(Object.keys(files).length, 1);
42+
const { file } = files;
43+
44+
assert.strictEqual(file.size, 301);
45+
assert.ok(file.path === undefined);
46+
47+
const dirFiles = fs.readdirSync(DEFAULT_UPLOAD_DIR);
48+
assert.ok(dirFiles.length === 0);
49+
50+
const uploadedFileStats = fs.statSync(CUSTOM_UPLOAD_FILE_PATH);
51+
assert.ok(uploadedFileStats.size === file.size);
52+
53+
fs.unlinkSync(CUSTOM_UPLOAD_FILE_PATH);
54+
res.end();
55+
server.close();
56+
});
57+
});
58+
59+
server.listen(PORT, (err) => {
60+
const choosenPort = server.address().port;
61+
const url = `http://localhost:${choosenPort}`;
62+
console.log('Server up and running at:', url);
63+
64+
assert(!err, 'should not have error, but be falsey');
65+
66+
const request = http.request({
67+
port: PORT,
68+
method: 'POST',
69+
headers: {
70+
'Content-Type': 'application/octet-stream',
71+
},
72+
});
73+
74+
fs.createReadStream(testFilePath).pipe(request);
75+
});

‎test/integration/test-store-files-option.js

+14-6
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ const assert = require('assert');
99
const formidable = require('../../src/index');
1010

1111
const PORT = 13532;
12-
const UPLOAD_DIR = path.join(os.tmpdir(), 'test-store-files-option');
12+
const DEFAULT_UPLOAD_DIR = path.join(
13+
os.tmpdir(),
14+
'test-store-files-option-default',
15+
);
16+
const CUSTOM_UPLOAD_FILE_PATH = path.join(DEFAULT_UPLOAD_DIR, 'test-file');
1317
const testFilePath = path.join(
1418
path.dirname(__dirname),
1519
'fixture',
@@ -18,10 +22,13 @@ const testFilePath = path.join(
1822
);
1923

2024
const server = http.createServer((req, res) => {
21-
if (!fs.existsSync(UPLOAD_DIR)) {
22-
fs.mkdirSync(UPLOAD_DIR);
25+
if (!fs.existsSync(DEFAULT_UPLOAD_DIR)) {
26+
fs.mkdirSync(DEFAULT_UPLOAD_DIR);
2327
}
24-
const form = formidable({ storeFiles: false, uploadDir: UPLOAD_DIR });
28+
const form = formidable({
29+
uploadDir: DEFAULT_UPLOAD_DIR,
30+
fileWriteStreamHandler: () => new fs.WriteStream(CUSTOM_UPLOAD_FILE_PATH),
31+
});
2532

2633
form.parse(req, (err, fields, files) => {
2734
assert.strictEqual(Object.keys(files).length, 1);
@@ -30,9 +37,10 @@ const server = http.createServer((req, res) => {
3037
assert.strictEqual(file.size, 301);
3138
assert.ok(file.path === undefined);
3239

33-
const dirFiles = fs.readdirSync(UPLOAD_DIR);
34-
assert.ok(dirFiles.length === 0);
40+
const uploadedFileStats = fs.statSync(CUSTOM_UPLOAD_FILE_PATH);
41+
assert.ok(uploadedFileStats.size === file.size);
3542

43+
fs.unlinkSync(CUSTOM_UPLOAD_FILE_PATH);
3644
res.end();
3745
server.close();
3846
});

0 commit comments

Comments
 (0)