UTF8: add to support string storage

Allow storage of a UTF8 string without terminating NUL, in a blob that
may be length-constrained.

Closes pabigot#21.

Author: pabigot

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+* **API** Add [UTF8][doc:UTF8] to encode UTF strings in a possibly
+  bounded buffer, resolving [issue #21][issue#21].
 * **API** Allow the layout parameter of
   a [VariantLayout][doc:VariantLayout] to be omitted in cases where no
   data beyond the discriminator is required,
@@ -170,6 +172,7 @@
 [doc:Union]: http://pabigot.github.io/buffer-layout/module-Layout-Union.html
 [doc:Union.getSourceVariant]: http://pabigot.github.io/buffer-layout/module-Layout-Union.html#getSourceVariant
 [doc:UnionDiscriminator]: http://pabigot.github.io/buffer-layout/module-Layout-UnionDiscriminator.html
+[doc:UTF8]: http://pabigot.github.io/buffer-layout/module-Layout-UTF8.html
 [doc:VariantLayout]: http://pabigot.github.io/buffer-layout/module-Layout-VariantLayout.html
 [issue#1]: https://github.com/pabigot/buffer-layout/issues/1
 [issue#2]: https://github.com/pabigot/buffer-layout/issues/2
@@ -189,6 +192,7 @@
 [issue#17]: https://github.com/pabigot/buffer-layout/issues/17
 [issue#19]: https://github.com/pabigot/buffer-layout/issues/19
 [issue#20]: https://github.com/pabigot/buffer-layout/issues/20
+[issue#21]: https://github.com/pabigot/buffer-layout/issues/21
 [ci:travis]: https://travis-ci.org/pabigot/buffer-layout
 [ci:coveralls]: https://coveralls.io/github/pabigot/buffer-layout
 [node:issue#3992]: https://github.com/nodejs/node/issues/3992

lib/Layout.js

@@ -1114,8 +1114,7 @@ class Sequence extends Layout {
     }
     const elo = this.elementLayout;
     const span = src.reduce((span, v) => {
-      elo.encode(v, b, offset + span);
-      return span + elo.getSpan(b, offset + span);
+      return span + elo.encode(v, b, offset + span);
     }, 0);
     if (this.count instanceof ExternalLayout) {
       this.count.encode(src.length, b, offset);
@@ -2406,6 +2405,100 @@ class CString extends Layout {
   }
 }
 
+/**
+ * Contain a UTF8 string with implicit length.
+ *
+ * *Factory*: {@link module:Layout.utf8|utf8}
+ *
+ * **NOTE** Because the length is implicit in the size of the buffer
+ * this layout should be used only in isolation, or in a situation
+ * where the length can be expressed by operating on a slice of the
+ * containing buffer.
+ *
+ * @param {Number} [maxSpan] - the maximum length allowed for encoded
+ * string content.  If not provided there is no bound on the allowed
+ * content.
+ *
+ * @param {String} [property] - initializer for {@link
+ * Layout#property|property}.
+ *
+ * @augments {Layout}
+ */
+class UTF8 extends Layout {
+  constructor(maxSpan, property) {
+    if (('string' === typeof maxSpan)
+        && (undefined === property)) {
+      property = maxSpan;
+      maxSpan = undefined;
+    }
+    if (undefined === maxSpan) {
+      maxSpan = -1;
+    } else if (!Number.isInteger(maxSpan)) {
+      throw new TypeError('maxSpan must be an integer');
+    }
+
+    super(-1, property);
+
+    /** The maximum span of the layout in bytes.
+     *
+     * Positive values are generally expected.  Zero is abnormal.
+     * Attempts to encode or decode a value that exceeds this length
+     * will throw a `RangeError`.
+     *
+     * A negative value indicates that there is no bound on the length
+     * of the content. */
+    this.maxSpan = maxSpan;
+  }
+
+  /** @override */
+  getSpan(b, offset) {
+    if (!(b instanceof Buffer)) {
+      throw new TypeError('b must be a Buffer');
+    }
+    if (undefined === offset) {
+      offset = 0;
+    }
+    return b.length - offset;
+  }
+
+  /** @override */
+  decode(b, offset, dest) {
+    if (undefined === offset) {
+      offset = 0;
+    }
+    let span = this.getSpan(b, offset);
+    if ((0 <= this.maxSpan)
+        && (this.maxSpan < span)) {
+      throw new RangeError('text length exceeds maxSpan');
+    }
+    return b.slice(offset, offset + span).toString('utf-8');
+  }
+
+  /** @override */
+  encode(src, b, offset) {
+    if (undefined === offset) {
+      offset = 0;
+    }
+    /* Must force this to a string, lest it be a number and the
+     * "utf8-encoding" below actually allocate a buffer of length
+     * src */
+    if ('string' !== typeof src) {
+      src = src.toString();
+    }
+    const srcb = new Buffer(src, 'utf8');
+    const span = srcb.length;
+    if ((0 <= this.maxSpan)
+        && (this.maxSpan < span)) {
+      throw new RangeError('text length exceeds maxSpan');
+    }
+    if ((offset + span) > b.length) {
+      throw new RangeError('encoding overruns Buffer');
+    }
+    srcb.copy(b, offset);
+    return span;
+  }
+}
+
 /**
  * Contain a constant value.
  *
@@ -2475,6 +2568,7 @@ exports.BitField = BitField;
 exports.Boolean = Boolean;
 exports.Blob = Blob;
 exports.CString = CString;
+exports.UTF8 = UTF8;
 exports.Constant = Constant;
 
 /** Factory for {@link GreedyCount}. */
@@ -2620,5 +2714,8 @@ exports.blob = ((length, property) => new Blob(length, property));
 /** Factory for {@link CString} values. */
 exports.cstr = (property => new CString(property));
 
+/** Factory for {@link UTF8} values. */
+exports.utf8 = ((maxSpan, property) => new UTF8(maxSpan, property));
+
 /** Factory for {@link Constant} values. */
 exports.const = ((value, property) => new Constant(value, property));

test/LayoutTest.js

@@ -1923,6 +1923,77 @@ suite('Layout', function() {
       assert.equal(Buffer.from('68690075006300', 'hex').compare(b), 0);
     });
   });
+  suite('UTF8', function() {
+    test('ctor', function() {
+      const cst = lo.utf8();
+      assert(0 > cst.span);
+      assert.strictEqual(cst.maxSpan, -1);
+    });
+    test('ctor with maxSpan', function() {
+      const cst = lo.utf8(5);
+      assert.strictEqual(cst.maxSpan, 5);
+    });
+    test('ctor with invalid maxSpan', function() {
+      assert.throws(() => new lo.UTF8(23.1), TypeError);
+    });
+    test('#getSpan', function() {
+      const cst = new lo.UTF8();
+      assert.throws(() => cst.getSpan(), TypeError);
+      assert.equal(cst.getSpan(Buffer.from('00', 'hex')), 1);
+      assert.equal(cst.getSpan(Buffer.from('4100', 'hex')), 2);
+      assert.equal(cst.getSpan(Buffer.from('4100', 'hex'), 1), 1);
+      assert.equal(cst.getSpan(Buffer.from('4142', 'hex')), 2);
+    });
+    test('#decode', function() {
+      const cst = new lo.UTF8(3);
+      assert.equal(cst.decode(Buffer.from('00', 'hex')), '\x00');
+      assert.equal(cst.decode(Buffer.from('4100', 'hex')), 'A\x00');
+      assert.equal(cst.decode(Buffer.from('4100', 'hex'), 1), '\x00');
+      assert.equal(cst.decode(Buffer.from('4142', 'hex')), 'AB');
+      assert.throws(() => cst.decode(Buffer.from('four', 'utf8')),
+                    RangeError);
+    });
+    test('#encode', function() {
+      const cst = new lo.UTF8();
+      const b = Buffer.alloc(3);
+      b.fill(0xFF);
+      assert.equal(cst.encode('', b), 0);
+      assert.equal(Buffer.from('ffffff', 'hex').compare(b), 0);
+      assert.equal(cst.encode('A', b), 1);
+      assert.equal(Buffer.from('41ffff', 'hex').compare(b), 0);
+      assert.equal(cst.encode('B', b, 1), 1);
+      assert.equal(Buffer.from('4142ff', 'hex').compare(b), 0);
+      assert.equal(cst.encode(5, b), 1);
+      assert.equal(Buffer.from('3542ff', 'hex').compare(b), 0);
+      assert.equal(cst.encode('abc', b), 3);
+      assert.equal(Buffer.from('616263', 'hex').compare(b), 0);
+      assert.throws(() => cst.encode('four', b), RangeError);
+    });
+    test('#encode with maxSpan', function() {
+      const cst = new lo.UTF8(2);
+      const b = Buffer.alloc(3);
+      b.fill(0xFF);
+      assert.throws(() => cst.encode('abc', b), RangeError);
+    });
+    test('in struct', function() {
+      const st = lo.struct([lo.utf8('k'),
+                            lo.utf8('v')]);
+      const b = Buffer.from('6162323334', 'hex');
+      assert.throws(() => st.getSpan(), RangeError);
+      assert.equal(st.fields[0].getSpan(b), b.length);
+      assert.equal(st.fields[1].getSpan(b, 2), b.length - 2);
+      assert.equal(st.getSpan(b), b.length);
+      assert.deepEqual(st.decode(b), {k: 'ab234', v: ''});
+    });
+    test('in seq', function() {
+      const seq = lo.seq(lo.utf8(), 3);
+      const b = Buffer.from('4162633435', 'hex');
+      assert.deepEqual(seq.decode(b), ['Abc45', '', '']);
+      b.fill(0xFF);
+      assert.equal(seq.encode(['hi', 'u', 'c'], b), 2 + 1 + 1);
+      assert.equal(Buffer.from('68697563ff', 'hex').compare(b), 0);
+    });
+  });
   suite('Constant', function() {
     test('ctor', function() {
       const c = new lo.Constant('value', 'p');