Skip to content

Commit 0961b07

Browse files
maciejhirszmaciejhirsz
authored
Merge pull request #32 from maciejhirsz/dev
0.7.4 closes #28, closes #29
2 parents 197c3ee + ba983e7 commit 0961b07

File tree

5 files changed

+193
-65
lines changed

5 files changed

+193
-65
lines changed

‎Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "json"
3-
version = "0.7.3"
3+
version = "0.7.4"
44
authors = ["Maciej Hirsz <[email protected]>"]
55
description = "JSON implementation in Rust"
66
repository = "https://github.com/maciejhirsz/json-rust"

‎src/codegen.rs

+5
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ pub trait Generator {
5151
self.write_char(b'-');
5252
}
5353

54+
if num > 1e19 || num < 1e-15 {
55+
self.write(format!("{:e}", num).as_bytes());
56+
return;
57+
}
58+
5459
let start = self.current_index();
5560

5661
self.write_digits_from_u64(num as u64);

‎src/error.rs

+41-25
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,18 @@ use std::error::Error;
33
use std::fmt;
44
use std::char;
55

6-
#[derive(Debug)]
6+
#[derive(Debug, PartialEq)]
77
pub enum JsonError {
8-
UnexpectedToken(String),
9-
UnexpectedCharacter(char),
8+
UnexpectedToken {
9+
token: String,
10+
line: usize,
11+
column: usize,
12+
},
13+
UnexpectedCharacter {
14+
ch: char,
15+
line: usize,
16+
column: usize,
17+
},
1018
UnexpectedEndOfJson,
1119
FailedUtf8Parsing,
1220
ArrayIndexOutOfBounds,
@@ -15,14 +23,12 @@ pub enum JsonError {
1523
}
1624

1725
impl JsonError {
18-
pub fn unexpected_token(token: Token) -> Self {
19-
JsonError::UnexpectedToken(format!("{:?}", token))
20-
}
21-
22-
pub fn unexpected_character(byte: u8) -> Self {
23-
JsonError::UnexpectedCharacter(
24-
char::from_u32(byte as u32).unwrap_or('?')
25-
)
26+
pub fn unexpected_token(token: Token,) -> Self {
27+
JsonError::UnexpectedToken {
28+
token: token.to_string(),
29+
line: 0,
30+
column: 0,
31+
}
2632
}
2733

2834
pub fn wrong_type(expected: &str) -> Self {
@@ -39,13 +45,23 @@ impl fmt::Display for JsonError {
3945
use JsonError::*;
4046

4147
match *self {
42-
UnexpectedToken(ref s) => write!(f, "Unexpected token: {}", s),
43-
UnexpectedCharacter(ch) => write!(f, "Unexpected character: {}", ch),
44-
UnexpectedEndOfJson => write!(f, "Unexpected end of JSON"),
45-
FailedUtf8Parsing => write!(f, "Failed to parse UTF-8 bytes"),
46-
ArrayIndexOutOfBounds => write!(f, "Array index out of bounds!"),
47-
WrongType(ref s) => write!(f, "Wrong type, expected: {}", s),
48-
UndefinedField(ref s) => write!(f, "Undefined field: {}", s)
48+
UnexpectedToken {
49+
ref token,
50+
ref line,
51+
ref column,
52+
} => write!(f, "Unexpected token: {} at ({}:{})", token, line, column),
53+
54+
UnexpectedCharacter {
55+
ref ch,
56+
ref line,
57+
ref column,
58+
} => write!(f, "Unexpected character: {} at ({}:{})", ch, line, column),
59+
60+
UnexpectedEndOfJson => write!(f, "Unexpected end of JSON"),
61+
FailedUtf8Parsing => write!(f, "Failed to parse UTF-8 bytes"),
62+
ArrayIndexOutOfBounds => write!(f, "Array index out of bounds!"),
63+
WrongType(ref s) => write!(f, "Wrong type, expected: {}", s),
64+
UndefinedField(ref s) => write!(f, "Undefined field: {}", s)
4965
}
5066
}
5167
}
@@ -54,13 +70,13 @@ impl Error for JsonError {
5470
fn description(&self) -> &str {
5571
use JsonError::*;
5672
match *self {
57-
UnexpectedToken(_) => "Unexpected token",
58-
UnexpectedCharacter(_) => "Unexpected character",
59-
UnexpectedEndOfJson => "Unexpected end of JSON",
60-
FailedUtf8Parsing => "Failed to read bytes as UTF-8 from JSON",
61-
ArrayIndexOutOfBounds => "Array index out of bounds!",
62-
WrongType(_) => "Wrong type",
63-
UndefinedField(_) => "Undefined field",
73+
UnexpectedToken { .. } => "Unexpected token",
74+
UnexpectedCharacter { .. } => "Unexpected character",
75+
UnexpectedEndOfJson => "Unexpected end of JSON",
76+
FailedUtf8Parsing => "Failed to read bytes as UTF-8 from JSON",
77+
ArrayIndexOutOfBounds => "Array index out of bounds!",
78+
WrongType(_) => "Wrong type",
79+
UndefinedField(_) => "Undefined field",
6480
}
6581
}
6682
}

‎src/parser.rs

+103-32
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,30 @@ pub enum Token {
1919
Null,
2020
}
2121

22+
impl Token {
23+
pub fn to_string(&self) -> String {
24+
match *self {
25+
Token::Comma => ",",
26+
Token::Colon => ":",
27+
Token::BracketOn => "[",
28+
Token::BracketOff => "]",
29+
Token::BraceOn => "{",
30+
Token::BraceOff => "}",
31+
Token::String(_) => "[string]",
32+
Token::Number(_) => "[number]",
33+
Token::Boolean(true) => "true",
34+
Token::Boolean(false) => "false",
35+
Token::Null => "null",
36+
}.into()
37+
}
38+
}
39+
2240
macro_rules! sequence {
2341
($tok:ident, $( $ch:pat ),*) => {
2442
$(
2543
match $tok.next_byte() {
2644
Some($ch) => {},
27-
Some(ch) => return Err(JsonError::unexpected_character(ch)),
45+
Some(ch) => return Err($tok.unexpected_character_error(ch)),
2846
None => return Err(JsonError::UnexpectedEndOfJson)
2947
}
3048
)*
@@ -48,8 +66,14 @@ macro_rules! read_num {
4866
}
4967
}
5068

69+
struct Position {
70+
pub line: usize,
71+
pub column: usize,
72+
}
73+
5174
struct Tokenizer<'a> {
52-
source: Enumerate<Bytes<'a>>,
75+
source: &'a str,
76+
byte_iter: Enumerate<Bytes<'a>>,
5377
buffer: Vec<u8>,
5478
left_over: Option<u8>,
5579
current_index: usize,
@@ -58,16 +82,39 @@ struct Tokenizer<'a> {
5882
impl<'a> Tokenizer<'a> {
5983
pub fn new(source: &'a str) -> Self {
6084
Tokenizer {
61-
source: source.bytes().enumerate(),
85+
source: source,
86+
byte_iter: source.bytes().enumerate(),
6287
buffer: Vec::with_capacity(512),
6388
left_over: None,
6489
current_index: 0,
6590
}
6691
}
6792

93+
pub fn source_position_from_index(&self, index: usize) -> Position {
94+
let (bytes, _) = self.source.split_at(index);
95+
96+
Position {
97+
line: bytes.lines().count(),
98+
column: bytes.lines().last().map(|line| {
99+
line.chars().count() + 1
100+
}).unwrap_or(1)
101+
}
102+
}
103+
104+
fn unexpected_character_error(&self, byte: u8) -> JsonError {
105+
let pos = self.source_position_from_index(self.current_index);
106+
let ch = char::from_u32(byte as u32).unwrap_or('?');
107+
108+
JsonError::UnexpectedCharacter {
109+
ch: ch,
110+
line: pos.line,
111+
column: pos.column,
112+
}
113+
}
114+
68115
#[inline(always)]
69116
fn next_byte(&mut self) -> Option<u8> {
70-
self.source.next().map(|(index, byte)| {
117+
self.byte_iter.next().map(|(index, byte)| {
71118
self.current_index = index;
72119
byte
73120
})
@@ -109,7 +156,7 @@ impl<'a> Tokenizer<'a> {
109156
b'0' ... b'9' => (ch - b'0') as u32,
110157
b'a' ... b'f' => (ch + 10 - b'a') as u32,
111158
b'A' ... b'F' => (ch + 10 - b'A') as u32,
112-
ch => return Err(JsonError::unexpected_character(ch)),
159+
ch => return Err(self.unexpected_character_error(ch)),
113160
})
114161
}
115162

@@ -219,10 +266,12 @@ impl<'a> Tokenizer<'a> {
219266
Ok(if first == b'-' { num * -1.0 } else { num })
220267
}
221268

222-
fn next(&mut self) -> JsonResult<Token> {
269+
fn next(&mut self) -> JsonResult<(usize, Token)> {
223270
loop {
224271
let ch = try!(self.checked_expect_byte());
225-
return Ok(match ch {
272+
let index = self.current_index;
273+
274+
return Ok((index, match ch {
226275
b',' => Token::Comma,
227276
b':' => Token::Colon,
228277
b'[' => Token::BracketOn,
@@ -245,25 +294,29 @@ impl<'a> Tokenizer<'a> {
245294
},
246295
// whitespace
247296
9 ... 13 | 32 | 133 | 160 => continue,
248-
_ => return Err(JsonError::unexpected_character(ch))
249-
});
297+
_ => return Err(self.unexpected_character_error(ch))
298+
}));
250299
}
251300
}
252301
}
253302

254303
macro_rules! expect {
255304
($parser:ident, $token:pat => $value:ident) => (
256305
match $parser.tokenizer.next() {
257-
Ok($token) => $value,
258-
Ok(token) => return Err(JsonError::unexpected_token(token)),
259-
Err(error) => return Err(error),
306+
Ok((_, $token)) => $value,
307+
Ok((index, token)) => {
308+
return Err($parser.unexpected_token_error(index, token));
309+
},
310+
Err(error) => return Err(error),
260311
}
261312
);
262313
($parser:ident, $token:pat) => ({
263314
match $parser.tokenizer.next() {
264-
Ok($token) => {},
265-
Ok(token) => return Err(JsonError::unexpected_token(token)),
266-
Err(error) => return Err(error),
315+
Ok((_, $token)) => {},
316+
Ok((index, token)) => {
317+
return Err($parser.unexpected_token_error(index, token));
318+
},
319+
Err(error) => return Err(error),
267320
}
268321
})
269322
}
@@ -279,10 +332,20 @@ impl<'a> Parser<'a> {
279332
}
280333
}
281334

335+
fn unexpected_token_error(&self, index: usize, token: Token) -> JsonError {
336+
let pos = self.tokenizer.source_position_from_index(index);
337+
338+
JsonError::UnexpectedToken {
339+
token: token.to_string(),
340+
line: pos.line,
341+
column: pos.column,
342+
}
343+
}
344+
282345
#[must_use]
283346
fn ensure_end(&mut self) -> JsonResult<()> {
284347
match self.tokenizer.next() {
285-
Ok(token) => Err(JsonError::unexpected_token(token)),
348+
Ok((index, token)) => Err(self.unexpected_token_error(index, token)),
286349
Err(JsonError::UnexpectedEndOfJson) => Ok(()),
287350
Err(error) => Err(error)
288351
}
@@ -292,18 +355,22 @@ impl<'a> Parser<'a> {
292355
let mut array = Vec::with_capacity(20);
293356

294357
match try!(self.tokenizer.next()) {
295-
Token::BracketOff => return Ok(JsonValue::Array(array)),
296-
token => array.push(try!(self.value_from(token))),
358+
(_, Token::BracketOff) => return Ok(JsonValue::Array(array)),
359+
(index, token) => {
360+
array.push(try!(self.value_from(index, token)));
361+
}
297362
}
298363

299364
loop {
300365
match try!(self.tokenizer.next()) {
301-
Token::Comma => {
366+
(_, Token::Comma) => {
302367
array.push(try!(self.value()));
303368
continue
304369
},
305-
Token::BracketOff => break,
306-
token => return Err(JsonError::unexpected_token(token))
370+
(_, Token::BracketOff) => break,
371+
(index, token) => {
372+
return Err(self.unexpected_token_error(index, token))
373+
}
307374
}
308375
}
309376

@@ -314,49 +381,53 @@ impl<'a> Parser<'a> {
314381
let mut object = BTreeMap::new();
315382

316383
match try!(self.tokenizer.next()) {
317-
Token::BraceOff => return Ok(JsonValue::Object(object)),
318-
Token::String(key) => {
384+
(_, Token::BraceOff) => return Ok(JsonValue::Object(object)),
385+
(_, Token::String(key)) => {
319386
expect!(self, Token::Colon);
320387
object.insert(key, try!(self.value()));
321388
},
322-
token => return Err(JsonError::unexpected_token(token))
389+
(index, token) => {
390+
return Err(self.unexpected_token_error(index, token))
391+
}
323392
}
324393

325394
loop {
326395
match try!(self.tokenizer.next()) {
327-
Token::Comma => {
396+
(_, Token::Comma) => {
328397
let key = expect!(self,
329398
Token::String(key) => key
330399
);
331400
expect!(self, Token::Colon);
332401
object.insert(key, try!(self.value()));
333402
continue
334403
},
335-
Token::BraceOff => break,
336-
token => return Err(JsonError::unexpected_token(token))
404+
(_, Token::BraceOff) => break,
405+
(index, token) => {
406+
return Err(self.unexpected_token_error(index, token))
407+
}
337408
}
338409
}
339410

340411
Ok(JsonValue::Object(object))
341412
}
342413

343-
fn value_from(&mut self, token: Token) -> JsonResult<JsonValue> {
414+
fn value_from(&mut self, index: usize, token: Token) -> JsonResult<JsonValue> {
344415
Ok(match token {
345416
Token::String(value) => JsonValue::String(value),
346417
Token::Number(value) => JsonValue::Number(value),
347418
Token::Boolean(value) => JsonValue::Boolean(value),
348419
Token::Null => JsonValue::Null,
349420
Token::BracketOn => return self.array(),
350421
Token::BraceOn => return self.object(),
351-
token => {
352-
return Err(JsonError::unexpected_token(token))
422+
_ => {
423+
return Err(self.unexpected_token_error(index, token))
353424
}
354425
})
355426
}
356427

357428
fn value(&mut self) -> JsonResult<JsonValue> {
358-
let token = try!(self.tokenizer.next());
359-
self.value_from(token)
429+
let (index, token) = try!(self.tokenizer.next());
430+
self.value_from(index, token)
360431
}
361432
}
362433

0 commit comments

Comments
 (0)