Support CodeUnits (#60)

Author: bicycle1885

src/TranscodingStreams.jl

@@ -5,6 +5,8 @@ export
     Noop,
     NoopStream
 
+const ByteData = Union{Vector{UInt8},Base.CodeUnits{UInt8}}
+
 include("memory.jl")
 include("buffer.jl")
 include("error.jl")

src/buffer.jl

@@ -35,6 +35,10 @@ mutable struct Buffer
     end
 end
 
+function Buffer(data::Base.CodeUnits{UInt8})
+    return Buffer(Vector{UInt8}(data))
+end
+
 function Base.length(buf::Buffer)
     return length(buf.data)
 end

src/memory.jl

@@ -11,7 +11,7 @@ struct Memory
     size::UInt
 end
 
-function Memory(data::Vector{UInt8})
+function Memory(data::ByteData)
     return Memory(pointer(data), sizeof(data))
 end
 

src/noop.jl

@@ -107,10 +107,14 @@ function Base.unsafe_write(stream::NoopStream, input::Ptr{UInt8}, nbytes::UInt)
     end
 end
 
-function Base.transcode(::Noop, data::Vector{UInt8})
+function Base.transcode(::Type{Noop}, data::ByteData)
+    return transcode(Noop(), data)
+end
+
+function Base.transcode(::Noop, data::ByteData)
     # Copy data because the caller may expect the return object is not the same
     # as from the input.
-    return copy(data)
+    return Vector{UInt8}(data)
 end
 
 

src/stream.jl

@@ -361,7 +361,7 @@ Insert `data` to the current reading position of `stream`.
 The next `read(stream, sizeof(data))` call will read data that are just
 inserted.
 """
-function unread(stream::TranscodingStream, data::Vector{UInt8})
+function unread(stream::TranscodingStream, data::ByteData)
     unsafe_unread(stream, pointer(data), sizeof(data))
 end
 

src/transcode.jl

@@ -27,7 +27,7 @@ julia> String(decompressed)
 
 ```
 """
-function Base.transcode(::Type{C}, data::Vector{UInt8}) where C<:Codec
+function Base.transcode(::Type{C}, data::ByteData) where C<:Codec
     codec = C()
     initialize(codec)
     try
@@ -70,7 +70,7 @@ julia> String(decompressed)
 
 ```
 """
-function Base.transcode(codec::Codec, data::Vector{UInt8})
+function Base.transcode(codec::Codec, data::ByteData)
     # Add `minoutsize` because `transcode` will be called at least two times.
     buffer2 = Buffer(
         expectedsize(codec, Memory(data)) + minoutsize(codec, Memory(C_NULL, 0)))

test/codecnoop.jl

@@ -9,7 +9,7 @@
     source = IOBuffer("foo")
     stream = TranscodingStream(Noop(), source)
     @test !eof(stream)
-    @test read(stream) == bb"foo"
+    @test read(stream) == b"foo"
     close(stream)
 
     data = rand(UInt8, 100_000)
@@ -28,15 +28,15 @@
     @test read(stream, UInt8) === UInt8('f')
     data = Vector{UInt8}(undef, 5)
     unsafe_read(stream, pointer(data), 5) === nothing
-    @test data == bb"oobar"
+    @test data == b"oobar"
     close(stream)
 
     sink = IOBuffer()
     stream = TranscodingStream(Noop(), sink)
     @test write(stream, "foo") === 3
     @test occursin("mode=write", repr(stream))
     flush(stream)
-    @test take!(sink) == bb"foo"
+    @test take!(sink) == b"foo"
     close(stream)
 
     data = rand(UInt8, 100_000)
@@ -49,7 +49,7 @@
     @test take!(sink) == data
     close(stream)
 
-    stream = TranscodingStream(Noop(), IOBuffer(bb"foobarbaz"))
+    stream = TranscodingStream(Noop(), IOBuffer(b"foobarbaz"))
     @test position(stream) === 0
     read(stream, UInt8)
     @test position(stream) === 1
@@ -71,13 +71,13 @@
     @test !ismarked(stream)
     close(stream)
 
-    stream = TranscodingStream(Noop(), IOBuffer(bb"foobarbaz"))
+    stream = TranscodingStream(Noop(), IOBuffer(b"foobarbaz"))
     seek(stream, 2)
-    @test read(stream, 3) == bb"oba"
+    @test read(stream, 3) == b"oba"
     seek(stream, 0)
-    @test read(stream, 3) == bb"foo"
+    @test read(stream, 3) == b"foo"
     seekstart(stream)
-    @test read(stream, 3) == bb"foo"
+    @test read(stream, 3) == b"foo"
     seekend(stream)
     @test eof(stream)
     close(stream)
@@ -116,7 +116,7 @@
     out = zeros(UInt8, 3)
     @test bytesavailable(stream) == 0
     @test TranscodingStreams.unsafe_read(stream, pointer(out), 10) == 3
-    @test out == bb"foo"
+    @test out == b"foo"
     close(stream)
 
     data = rand(UInt8, 1999)
@@ -132,11 +132,11 @@
     close(stream)
 
     stream = NoopStream(NoopStream(IOBuffer("foobar")))
-    @test read(stream) == bb"foobar"
+    @test read(stream) == b"foobar"
     close(stream)
 
     stream = NoopStream(NoopStream(NoopStream(IOBuffer("foobar"))))
-    @test read(stream) == bb"foobar"
+    @test read(stream) == b"foobar"
     close(stream)
 
     # Two buffers are the same object.
@@ -151,7 +151,7 @@
     @test s1.state.buffer1 === s2.state.buffer1 === s3.state.buffer1 ===
           s1.state.buffer2 === s2.state.buffer2 === s3.state.buffer2
 
-    stream = TranscodingStream(Noop(), IOBuffer(bb"foobar"))
+    stream = TranscodingStream(Noop(), IOBuffer(b"foobar"))
     @test TranscodingStreams.stats(stream).in === Int64(0)
     @test TranscodingStreams.stats(stream).out === Int64(0)
     read(stream)
@@ -162,7 +162,7 @@
     stream = TranscodingStream(Noop(), IOBuffer())
     @test TranscodingStreams.stats(stream).in === Int64(0)
     @test TranscodingStreams.stats(stream).out === Int64(0)
-    write(stream, bb"foobar")
+    write(stream, b"foobar")
     flush(stream)
     @test TranscodingStreams.stats(stream).in === Int64(6)
     @test TranscodingStreams.stats(stream).out === Int64(6)
@@ -176,23 +176,23 @@
 
     stream = NoopStream(IOBuffer("foobar"))
     @test bytesavailable(stream) === 0
-    @test readavailable(stream) == bb""
+    @test readavailable(stream) == b""
     @test read(stream, UInt8) === UInt8('f')
     @test bytesavailable(stream) === 5
-    @test readavailable(stream) == bb"oobar"
+    @test readavailable(stream) == b"oobar"
     close(stream)
 
-    data = bb""
+    data = b""
     @test transcode(Noop, data)  == data
     @test transcode(Noop, data) !== data
-    data = bb"foo"
+    data = b"foo"
     @test transcode(Noop, data)  == data
     @test transcode(Noop, data) !== data
 
-    data = bb""
+    data = b""
     @test transcode(Noop(), data)  == data
     @test transcode(Noop(), data) !== data
-    data = bb"foo"
+    data = b"foo"
     @test transcode(Noop(), data)  == data
     @test transcode(Noop(), data) !== data
 
@@ -202,56 +202,56 @@
     TranscodingStreams.test_roundtrip_lines(NoopStream, NoopStream)
 
     # switch write => read
-    stream = NoopStream(IOBuffer(bb"foobar", read=true, write=true))
+    stream = NoopStream(IOBuffer(b"foobar", read=true, write=true))
     @test_throws ArgumentError begin
-        write(stream, bb"xyz")
+        write(stream, b"xyz")
         read(stream, 3)
     end
 
     # switch read => write
-    stream = NoopStream(IOBuffer(bb"foobar", read=true, write=true))
+    stream = NoopStream(IOBuffer(b"foobar", read=true, write=true))
     @test_throws ArgumentError begin
         read(stream, 3)
-        write(stream, bb"xyz")
+        write(stream, b"xyz")
     end
 
     stream = NoopStream(IOBuffer(""))
-    @test TranscodingStreams.unread(stream, bb"foo") === nothing
-    @test read(stream, 3) == bb"foo"
+    @test TranscodingStreams.unread(stream, b"foo") === nothing
+    @test read(stream, 3) == b"foo"
     close(stream)
 
     stream = NoopStream(IOBuffer("foo"))
-    @test read(stream, 3) == bb"foo"
-    @test TranscodingStreams.unread(stream, bb"bar") === nothing
-    @test read(stream, 3) == bb"bar"
+    @test read(stream, 3) == b"foo"
+    @test TranscodingStreams.unread(stream, b"bar") === nothing
+    @test read(stream, 3) == b"bar"
     close(stream)
 
     stream = NoopStream(IOBuffer("foobar"))
-    @test TranscodingStreams.unread(stream, bb"baz") === nothing
-    @test read(stream, 3) == bb"baz"
-    @test read(stream, 3) == bb"foo"
-    @test read(stream, 3) == bb"bar"
+    @test TranscodingStreams.unread(stream, b"baz") === nothing
+    @test read(stream, 3) == b"baz"
+    @test read(stream, 3) == b"foo"
+    @test read(stream, 3) == b"bar"
     @test eof(stream)
     close(stream)
 
     stream = NoopStream(IOBuffer("foobar"))
-    @test read(stream, 3) == bb"foo"
-    @test TranscodingStreams.unread(stream, bb"baz") === nothing
-    @test read(stream, 3) == bb"baz"
-    @test read(stream, 3) == bb"bar"
+    @test read(stream, 3) == b"foo"
+    @test TranscodingStreams.unread(stream, b"baz") === nothing
+    @test read(stream, 3) == b"baz"
+    @test read(stream, 3) == b"bar"
     @test eof(stream)
     close(stream)
 
     stream = NoopStream(IOBuffer("foobar"))
-    @test read(stream, 3) == bb"foo"
-    @test read(stream, 3) == bb"bar"
-    @test TranscodingStreams.unread(stream, bb"baz") === nothing
-    @test read(stream, 3) == bb"baz"
+    @test read(stream, 3) == b"foo"
+    @test read(stream, 3) == b"bar"
+    @test TranscodingStreams.unread(stream, b"baz") === nothing
+    @test read(stream, 3) == b"baz"
     @test eof(stream)
     close(stream)
 
     stream = NoopStream(IOBuffer("foobar"))
-    @test_throws ArgumentError TranscodingStreams.unsafe_unread(stream, pointer(bb"foo"), -1)
+    @test_throws ArgumentError TranscodingStreams.unsafe_unread(stream, pointer(b"foo"), -1)
     close(stream)
 
     stream = NoopStream(IOBuffer(""))

test/codecquadruple.jl

@@ -29,12 +29,12 @@ function TranscodingStreams.minoutsize(
 end
 
 @testset "Quadruple Codec" begin
-    @test transcode(QuadrupleCodec, bb"") == bb""
-    @test transcode(QuadrupleCodec, bb"a") == bb"aaaa"
-    @test transcode(QuadrupleCodec, bb"ab") == bb"aaaabbbb"
-    @test transcode(QuadrupleCodec(), bb"") == bb""
-    @test transcode(QuadrupleCodec(), bb"a") == bb"aaaa"
-    @test transcode(QuadrupleCodec(), bb"ab") == bb"aaaabbbb"
+    @test transcode(QuadrupleCodec, b"") == b""
+    @test transcode(QuadrupleCodec, b"a") == b"aaaa"
+    @test transcode(QuadrupleCodec, b"ab") == b"aaaabbbb"
+    @test transcode(QuadrupleCodec(), b"") == b""
+    @test transcode(QuadrupleCodec(), b"a") == b"aaaa"
+    @test transcode(QuadrupleCodec(), b"ab") == b"aaaabbbb"
 
     #=
     data = "x"^1024
@@ -43,11 +43,11 @@ end
     =#
 
     stream = TranscodingStream(QuadrupleCodec(), NoopStream(IOBuffer("foo")))
-    @test read(stream) == bb"ffffoooooooo"
+    @test read(stream) == b"ffffoooooooo"
     close(stream)
 
     stream = NoopStream(TranscodingStream(QuadrupleCodec(), NoopStream(IOBuffer("foo"))))
-    @test read(stream) == bb"ffffoooooooo"
+    @test read(stream) == b"ffffoooooooo"
     close(stream)
 
     # Buffers are shared.

test/runtests.jl

@@ -13,17 +13,12 @@ import TranscodingStreams:
     #=ismarked,=# mark!, unmark!, reset!,
     makemargin!, emptybuffer!
 
-# binary bytes
-macro bb_str(data)
-    convert(Vector{UInt8}, codeunits(data))
-end
-
 @testset "Buffer" begin
     buf = Buffer(1024)
     @test buf isa Buffer
     @test length(buf.data) == 1024
 
-    data = bb"foobar"
+    data = Vector{UInt8}(b"foobar")
     buf = Buffer(data)
     @test buf isa Buffer
     @test bufferptr(buf) === pointer(data)
@@ -69,7 +64,7 @@ end
 end
 
 @testset "Memory" begin
-    data = bb"foobar"
+    data = Vector{UInt8}(b"foobar")
     mem = TranscodingStreams.Memory(pointer(data), sizeof(data))
     @test mem isa TranscodingStreams.Memory
     @test mem.ptr === pointer(data)
@@ -90,7 +85,7 @@ end
     @test_throws BoundsError mem[7] = 0x00
     @test_throws BoundsError mem[0] = 0x00
 
-    data = bb"foobar"
+    data = Vector{UInt8}(b"foobar")
     mem = TranscodingStreams.Memory(data)
     @test mem isa TranscodingStreams.Memory
     @test mem.ptr == pointer(data)