Skip to content

Commit 799f10c

Browse files
ryancdotorgryancdotorg
committed
try to plumb ffi apis
1 parent 42a3826 commit 799f10c

File tree

8 files changed

+103
-24
lines changed

8 files changed

+103
-24
lines changed

‎c/arg.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,21 @@ size_t set_options(BrotliEncoderParameter *out_encoder_param_keys,
7070
out_encoder_param_values[ret] = 1;
7171
ret += 1;
7272
}
73+
if (strstr(argv[i], "-appendable") == argv[i]) {
74+
out_encoder_param_keys[ret] = BROTLI_PARAM_APPENDABLE;
75+
out_encoder_param_values[ret] = 1;
76+
ret += 1;
77+
}
78+
if (strstr(argv[i], "-bytealign") == argv[i]) {
79+
out_encoder_param_keys[ret] = BROTLI_PARAM_BYTE_ALIGN;
80+
out_encoder_param_values[ret] = 1;
81+
ret += 1;
82+
}
83+
if (strstr(argv[i], "-bare") == argv[i]) {
84+
out_encoder_param_keys[ret] = BROTLI_PARAM_BARE_STREAM;
85+
out_encoder_param_values[ret] = 1;
86+
ret += 1;
87+
}
7388
}
7489
return ret;
7590
}

‎c/brotli/encode.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
224224
BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
225225
BROTLI_PARAM_CATABLE = 167,
226226
BROTLI_PARAM_APPENDABLE = 168,
227-
BROTLI_PARAM_MAGIC_NUMBER = 169
227+
BROTLI_PARAM_MAGIC_NUMBER = 169,
228+
BROTLI_PARAM_NO_DICTIONARY = 170,
229+
BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
230+
BROTLI_PARAM_BYTE_ALIGN = 172,
231+
BROTLI_PARAM_BARE_STREAM = 173
228232
} BrotliEncoderParameter;
229233

230234
/**

‎c/go/brotli/brotli.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ type CompressionOptions struct {
4343
Catable bool
4444
Appendable bool
4545
Magic bool
46+
ByteAlign bool
47+
BareStream bool
4648
Mode int
4749
LgWin byte
4850
LgBlock byte
@@ -157,6 +159,14 @@ func makeCompressionOptionsStreams(options CompressionOptions,
157159
qualityParams = append(qualityParams, C.BROTLI_PARAM_MAGIC_NUMBER)
158160
values = append(values, 1)
159161
}
162+
if options.ByteAlign {
163+
qualityParams = append(qualityParams, C.BROTLI_PARAM_BYTE_ALIGN)
164+
values = append(values, 1)
165+
}
166+
if options.BareStream {
167+
qualityParams = append(qualityParams, C.BROTLI_PARAM_BARE_STREAM)
168+
values = append(values, 1)
169+
}
160170
if options.Mode != 0 {
161171
qualityParams = append(qualityParams, C.BROTLI_PARAM_MODE)
162172
values = append(values, C.uint32_t(options.Mode))

‎c/go/brotli/brotli/encode.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
224224
BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
225225
BROTLI_PARAM_CATABLE = 167,
226226
BROTLI_PARAM_APPENDABLE = 168,
227-
BROTLI_PARAM_MAGIC_NUMBER = 169
227+
BROTLI_PARAM_MAGIC_NUMBER = 169,
228+
BROTLI_PARAM_NO_DICTIONARY = 170,
229+
BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
230+
BROTLI_PARAM_BYTE_ALIGN = 172,
231+
BROTLI_PARAM_BARE_STREAM = 173,
228232
} BrotliEncoderParameter;
229233

230234
/**

‎c/py/brotli.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,10 @@ def BrotliParseHeader(raw_data):
313313
BROTLI_PARAM_CATABLE = 167
314314
BROTLI_PARAM_APPENDABLE = 168
315315
BROTLI_PARAM_MAGIC_NUMBER = 169
316+
BROTLI_PARAM_NO_DICTIONARY = 170
317+
BROTLI_PARAM_FAVOR_EFFICIENCY = 171
318+
BROTLI_PARAM_BYTE_ALIGN = 172
319+
BROTLI_PARAM_BARE_STREAM = 173
316320

317321
#simple test binary
318322
def main(args):

‎src/bin/brotli.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,7 @@ fn main() {
530530
}
531531
if (argument == "-bare" || argument == "--bare") && !double_dash {
532532
params.bare_stream = true;
533+
params.byte_align = true;
533534
continue;
534535
}
535536
if (argument == "-appendable" || argument == "--appendable") && !double_dash {
@@ -753,6 +754,14 @@ fn main() {
753754
}
754755
panic!("Unknown Argument {:}", argument);
755756
}
757+
if params.bare_stream && !params.appendable {
758+
println_stderr!("bare streams only supported when catable or appendable!");
759+
return;
760+
}
761+
if params.byte_align && !params.appendable {
762+
println_stderr!("byte aligned streams only supported when catable or appendable!");
763+
return;
764+
}
756765
if filenames[0] != "" {
757766
let mut input = match File::open(&Path::new(&filenames[0])) {
758767
Err(why) => panic!("couldn't open {:}\n{:}", filenames[0], why),

‎src/enc/brotli_bit_stream.rs

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2765,18 +2765,51 @@ pub fn BrotliStoreUncompressedMetaBlock<Cb, Alloc: BrotliAlloc>
27652765
}
27662766
}
27672767

2768-
27692768
pub fn BrotliStoreSyncMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
27702769
BrotliWriteBits(6u8, 6u64, storage_ix, storage);
27712770
JumpToByteBoundary(storage_ix, storage);
27722771
}
27732772

2773+
pub fn BrotliWritePaddingMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
2774+
if *storage_ix & 7 != 0 {
2775+
BrotliWriteBits(6u8, 6u64, storage_ix, storage);
2776+
JumpToByteBoundary(storage_ix, storage);
2777+
}
2778+
}
2779+
27742780
pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
27752781
BrotliWriteBits(1u8, 1u64, storage_ix, storage);
27762782
BrotliWriteBits(1u8, 1u64, storage_ix, storage);
27772783
JumpToByteBoundary(storage_ix, storage);
27782784
}
27792785

2786+
pub fn BrotliWriteRawMetadataMetaBlock(storage_ix: &mut usize, storage: &mut [u8], len: usize, data: &mut [u8]) {
2787+
BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last
2788+
BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1)
2789+
BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved
2790+
if len > 16777215 {
2791+
panic!("metadata too large");
2792+
} else if len > 65535 {
2793+
BrotliWriteBits(2u8, 3u64, storage_ix, storage);
2794+
BrotliWriteBits(8u8, ((len >> 16) & 255) as u64, storage_ix, storage);
2795+
BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage);
2796+
BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
2797+
} else if len > 255 {
2798+
BrotliWriteBits(2u8, 2u64, storage_ix, storage);
2799+
BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage);
2800+
BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
2801+
} else if len == 0 {
2802+
BrotliWriteBits(2u8, 0u64, storage_ix, storage);
2803+
} else {
2804+
BrotliWriteBits(2u8, 1u64, storage_ix, storage);
2805+
BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
2806+
}
2807+
JumpToByteBoundary(storage_ix, storage);
2808+
for index in 0..len {
2809+
BrotliWriteBits(8u8, data[index] as u64, storage_ix, storage);
2810+
}
2811+
}
2812+
27802813
const MAX_SIZE_ENCODING:usize = 10;
27812814

27822815
fn encode_base_128(mut value: u64)-> (usize, [u8;MAX_SIZE_ENCODING]) {

‎src/enc/encode.rs

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ use super::bit_cost::{BitsEntropy, ShannonEntropy};
1616
use super::block_split::BlockSplit;
1717
#[allow(unused_imports)]
1818
use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree,
19-
BrotliStoreMetaBlock, BrotliStoreSyncMetaBlock,
20-
BrotliStoreMetaBlockFast, BrotliStoreMetaBlockTrivial,
21-
BrotliStoreUncompressedMetaBlock,
22-
BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock,
19+
BrotliStoreMetaBlock, BrotliStoreMetaBlockFast,
20+
BrotliStoreMetaBlockTrivial, BrotliStoreUncompressedMetaBlock,
21+
BrotliWriteEmptyLastMetaBlock,
22+
BrotliWritePaddingMetaBlock, BrotliWriteMetadataMetaBlock,
2323
MetaBlockSplit, RecoderState, JumpToByteBoundary};
2424

2525
use enc::input_pair::InputReferenceMut;
@@ -740,7 +740,7 @@ fn EnsureInitialized<Alloc: BrotliAlloc>
740740
if (*s).params.quality == 0i32 || (*s).params.quality == 1i32 {
741741
lgwin = brotli_max_int(lgwin, 18i32);
742742
}
743-
if !(*s).params.bare_stream {
743+
if !((*s).params.catable && (*s).params.bare_stream) {
744744
EncodeWindowBits(lgwin, s.params.large_window, &mut (*s).last_bytes_, &mut (*s).last_bytes_bits_);
745745
}
746746
}
@@ -1973,6 +1973,15 @@ fn DecideOverLiteralContextModeling(input: &[u8],
19731973
literal_context_map);
19741974
}
19751975
}
1976+
fn WriteEmptyLastBlocksInternal(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) {
1977+
// insert empty block for byte alignment if required
1978+
if params.byte_align {
1979+
BrotliWritePaddingMetaBlock(storage_ix, storage);
1980+
}
1981+
if !params.bare_stream {
1982+
BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
1983+
}
1984+
}
19761985
fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
19771986
Cb>
19781987
(alloc: &mut Alloc,
@@ -2011,8 +2020,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
20112020
let literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
20122021
let mut block_params = params.clone();
20132022
if bytes == 0usize {
2014-
BrotliWriteBits(2usize, 3, storage_ix, storage);
2015-
*storage_ix = (*storage_ix).wrapping_add(7u32 as (usize)) & !7u32 as (usize);
2023+
WriteEmptyLastBlocksInternal(params, storage_ix, storage);
20162024
return;
20172025
}
20182026
if ShouldCompress(data,
@@ -2035,13 +2043,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
20352043
false,
20362044
cb);
20372045
if actual_is_last != is_last {
2038-
// insert empty block for byte alignment if required
2039-
if params.byte_align && ((*storage_ix & 7u32 as (usize)) != 0) {
2040-
BrotliStoreSyncMetaBlock(storage_ix, storage);
2041-
}
2042-
if !params.bare_stream {
2043-
BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
2044-
}
2046+
WriteEmptyLastBlocksInternal(params, storage_ix, storage);
20452047
}
20462048
return;
20472049
}
@@ -2181,13 +2183,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
21812183
cb);
21822184
}
21832185
if actual_is_last != is_last {
2184-
// insert empty block for byte alignment if required
2185-
if params.byte_align && ((*storage_ix & 7u32 as (usize)) != 0) {
2186-
BrotliStoreSyncMetaBlock(storage_ix, storage);
2187-
}
2188-
if !params.bare_stream {
2189-
BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
2190-
}
2186+
WriteEmptyLastBlocksInternal(params, storage_ix, storage);
21912187
}
21922188
}
21932189

@@ -2284,6 +2280,10 @@ fn EncodeData<Alloc: BrotliAlloc,
22842280
*out_size = catable_header_size;
22852281
s.is_first_mb = IsFirst::HeaderWritten;
22862282
}
2283+
// fixup for empty stream - note: catable is always appendable
2284+
if bytes == 0 && s.params.byte_align && s.params.appendable && !s.params.catable {
2285+
BrotliWritePaddingMetaBlock(&mut storage_ix, (*s).storage_.slice_mut());
2286+
}
22872287
}
22882288
if let IsFirst::BothCatableBytesWritten = s.is_first_mb {
22892289
// nothing to do here, move along

0 commit comments

Comments
 (0)