From 6ff5c445f37083c785111c904426c94b7a01f4b4 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 03:47:09 +0200 Subject: Use custom struct/field naming to deserialize spans --- test-suite/tests/spanned.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test-suite/tests/spanned.rs (limited to 'test-suite/tests') diff --git a/test-suite/tests/spanned.rs b/test-suite/tests/spanned.rs new file mode 100644 index 0000000..fec9ea8 --- /dev/null +++ b/test-suite/tests/spanned.rs @@ -0,0 +1,30 @@ +extern crate serde; +extern crate toml; +#[macro_use] +extern crate serde_derive; + +use toml::Spanned; +use std::collections::HashMap; + +#[test] +fn test_spanned_field() { + #[derive(Deserialize)] + struct Foo { + foo: Spanned, + } + + fn good<'de, T>(s: &'de str, expected: &str) where T: serde::Deserialize<'de> { + let foo: Foo = toml::from_str(s).unwrap(); + + assert_eq!(6, foo.foo.start); + assert_eq!(s.len(), foo.foo.end); + assert_eq!(expected, &s[foo.foo.start..foo.foo.end]); + } + + good::("foo = \"foo\"", "\"foo\""); + good::("foo = 42", "42"); + good::>( + "foo = {\"foo\" = 42, \"bar\" = 42}", + "{\"foo\" = 42, \"bar\" = 42}" + ); +} -- cgit v1.2.3 From 6c067202e061910d5018c4f96e9a7595c4291a08 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 04:05:05 +0200 Subject: Adjust spans for more complex types --- src/de.rs | 52 ++++++++++++++++++++++++++++----------------- src/tokens.rs | 29 ++++++++++++++++++------- test-suite/tests/spanned.rs | 13 ++++++++++++ 3 files changed, 67 insertions(+), 27 deletions(-) (limited to 'test-suite/tests') diff --git a/src/de.rs b/src/de.rs index cc1e2ca..4e2b4b7 100644 --- a/src/de.rs +++ b/src/de.rs @@ -821,16 +821,20 @@ impl<'a> Deserializer<'a> { Value { e: E::Boolean(false), start: start, end: end } } Some((span, Token::Keylike(key))) => self.number_or_date(span, key)?, - Some((_, Token::Plus)) => self.number_leading_plus()?, - Some((Span { start, end }, Token::LeftBrace)) => { - self.inline_table().map(|table| Value { + Some((span, Token::Plus)) => self.number_leading_plus(span)?, + Some((Span { start, .. }, Token::LeftBrace)) => { + self.inline_table().map(|(Span { end, .. }, table)| Value { e: E::InlineTable(table), start: start, end: end })? } - Some((Span { start, end }, Token::LeftBracket)) => { - self.array().map(|array| Value { e: E::Array(array), start: start, end: end })? + Some((Span { start, .. }, Token::LeftBracket)) => { + self.array().map(|(Span { end, .. }, array)| Value { + e: E::Array(array), + start: start, + end: end + })? } Some(token) => { return Err(self.error(at, ErrorKind::Wanted { @@ -882,11 +886,13 @@ impl<'a> Deserializer<'a> { } } - fn number_leading_plus(&mut self) -> Result, Error> { - let start = self.tokens.current(); + fn number_leading_plus(&mut self, Span { start, .. }: Span) -> Result, Error> { + let start_token = self.tokens.current(); match self.next()? { - Some((span, Token::Keylike(s))) => self.number(span, s), - _ => Err(self.error(start, ErrorKind::NumberInvalid)), + Some((Span { end, .. }, Token::Keylike(s))) => { + self.number(Span { start: start, end: end }, s) + }, + _ => Err(self.error(start_token, ErrorKind::NumberInvalid)), } } @@ -1042,11 +1048,11 @@ impl<'a> Deserializer<'a> { // TODO(#140): shouldn't buffer up this entire table in memory, it'd be // great to defer parsing everything until later. - fn inline_table(&mut self) -> Result, Value<'a>)>, Error> { + fn inline_table(&mut self) -> Result<(Span, Vec<(Cow<'a, str>, Value<'a>)>), Error> { let mut ret = Vec::new(); self.eat_whitespace()?; - if self.eat(Token::RightBrace)? { - return Ok(ret) + if let Some(span) = self.eat_spanned(Token::RightBrace)? { + return Ok((span, ret)) } loop { let key = self.table_key()?; @@ -1056,8 +1062,8 @@ impl<'a> Deserializer<'a> { ret.push((key, self.value()?)); self.eat_whitespace()?; - if self.eat(Token::RightBrace)? { - return Ok(ret) + if let Some(span) = self.eat_spanned(Token::RightBrace)? { + return Ok((span, ret)) } self.expect(Token::Comma)?; self.eat_whitespace()?; @@ -1066,7 +1072,7 @@ impl<'a> Deserializer<'a> { // TODO(#140): shouldn't buffer up this entire array in memory, it'd be // great to defer parsing everything until later. - fn array(&mut self) -> Result>, Error> { + fn array(&mut self) -> Result<(Span, Vec>), Error> { let mut ret = Vec::new(); let intermediate = |me: &mut Deserializer| { @@ -1081,8 +1087,8 @@ impl<'a> Deserializer<'a> { loop { intermediate(self)?; - if self.eat(Token::RightBracket)? { - return Ok(ret) + if let Some(span) = self.eat_spanned(Token::RightBracket)? { + return Ok((span, ret)) } let at = self.tokens.current(); let value = self.value()?; @@ -1098,8 +1104,8 @@ impl<'a> Deserializer<'a> { } } intermediate(self)?; - self.expect(Token::RightBracket)?; - Ok(ret) + let span = self.expect_spanned(Token::RightBracket)?; + Ok((span, ret)) } fn table_key(&mut self) -> Result, Error> { @@ -1122,10 +1128,18 @@ impl<'a> Deserializer<'a> { self.tokens.eat(expected).map_err(|e| self.token_error(e)) } + fn eat_spanned(&mut self, expected: Token<'a>) -> Result, Error> { + self.tokens.eat_spanned(expected).map_err(|e| self.token_error(e)) + } + fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> { self.tokens.expect(expected).map_err(|e| self.token_error(e)) } + fn expect_spanned(&mut self, expected: Token<'a>) -> Result { + self.tokens.expect_spanned(expected).map_err(|e| self.token_error(e)) + } + fn next(&mut self) -> Result)>, Error> { self.tokens.next().map_err(|e| self.token_error(e)) } diff --git a/src/tokens.rs b/src/tokens.rs index bcabd94..bfdc74e 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -118,21 +118,34 @@ impl<'a> Tokenizer<'a> { } pub fn eat(&mut self, expected: Token<'a>) -> Result { - match self.peek()? { - Some((_, ref found)) if expected == *found => {} - Some(_) => return Ok(false), - None => return Ok(false), - } + self.eat_spanned(expected).map(|s| s.is_some()) + } + + /// Eat a value, returning it's span if it was consumed. + pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result, Error> { + let span = match self.peek()? { + Some((span, ref found)) if expected == *found => span, + Some(_) => return Ok(None), + None => return Ok(None), + }; + drop(self.next()); - Ok(true) + Ok(Some(span)) } pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> { + // ignore span + let _ = self.expect_spanned(expected)?; + Ok(()) + } + + /// Expect the given token returning its span. + pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result { let current = self.current(); match self.next()? { - Some((_, found)) => { + Some((span, found)) => { if expected == found { - Ok(()) + Ok(span) } else { Err(Error::Wanted { at: current, diff --git a/test-suite/tests/spanned.rs b/test-suite/tests/spanned.rs index fec9ea8..c5dc28e 100644 --- a/test-suite/tests/spanned.rs +++ b/test-suite/tests/spanned.rs @@ -23,8 +23,21 @@ fn test_spanned_field() { good::("foo = \"foo\"", "\"foo\""); good::("foo = 42", "42"); + // leading plus + good::("foo = +42", "+42"); + // table good::>( "foo = {\"foo\" = 42, \"bar\" = 42}", "{\"foo\" = 42, \"bar\" = 42}" ); + // array + good::>( + "foo = [0, 1, 2, 3, 4]", + "[0, 1, 2, 3, 4]" + ); + // datetime + good::( + "foo = \"1997-09-09T09:09:09Z\"", + "\"1997-09-09T09:09:09Z\"" + ); } -- cgit v1.2.3 From 2b40d542a0ea645c5e3c132aabc0082dffa2f554 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 05:36:09 +0200 Subject: Fix incorrectly reported spans for Datetime --- src/de.rs | 36 +++++++++++++++++++++++------------- src/lib.rs | 1 + test-suite/tests/spanned.rs | 23 +++++++++++++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) (limited to 'test-suite/tests') diff --git a/src/de.rs b/src/de.rs index 5521e00..1d43cc9 100644 --- a/src/de.rs +++ b/src/de.rs @@ -853,16 +853,16 @@ impl<'a> Deserializer<'a> { { if s.contains('T') || (s.len() > 1 && s[1..].contains('-')) && !s.contains("e-") { - self.datetime(s, false).map(|d| Value { + self.datetime(span, s, false).map(|(Span { start, end }, d)| Value { e: E::Datetime(d), - start: span.start, - end: span.end + start: start, + end: end }) } else if self.eat(Token::Colon)? { - self.datetime(s, true).map(|d| Value { + self.datetime(span, s, true).map(|(Span { start, end }, d)| Value { e: E::Datetime(d), - start: span.start, - end: span.end + start: start, + end: end }) } else { self.number(span, s) @@ -1006,9 +1006,10 @@ impl<'a> Deserializer<'a> { }) } - fn datetime(&mut self, date: &'a str, colon_eaten: bool) - -> Result<&'a str, Error> { + fn datetime(&mut self, mut span: Span, date: &'a str, colon_eaten: bool) + -> Result<(Span, &'a str), Error> { let start = self.tokens.substr_offset(date); + if colon_eaten || self.eat(Token::Colon)? { // minutes match self.next()? { @@ -1018,13 +1019,17 @@ impl<'a> Deserializer<'a> { // Seconds self.expect(Token::Colon)?; match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } // Fractional seconds if self.eat(Token::Period)? { match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } @@ -1032,19 +1037,24 @@ impl<'a> Deserializer<'a> { // offset if self.eat(Token::Plus)? { match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } if self.eat(Token::Colon)? { match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } } + let end = self.tokens.current(); - Ok(&self.tokens.input()[start..end]) + Ok((span, &self.tokens.input()[start..end])) } // TODO(#140): shouldn't buffer up this entire table in memory, it'd be diff --git a/src/lib.rs b/src/lib.rs index b90612c..6fbf5d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -171,4 +171,5 @@ mod tokens; pub mod macros; pub mod spanned; +#[doc(no_inline)] pub use spanned::Spanned; diff --git a/test-suite/tests/spanned.rs b/test-suite/tests/spanned.rs index c5dc28e..4339a0c 100644 --- a/test-suite/tests/spanned.rs +++ b/test-suite/tests/spanned.rs @@ -4,8 +4,26 @@ extern crate toml; extern crate serde_derive; use toml::Spanned; +use toml::value::Datetime; use std::collections::HashMap; +/// A set of good datetimes. +pub fn good_datetimes() -> Vec<&'static str> { + let mut v = Vec::new(); + v.push("1997-09-09T09:09:09Z"); + v.push("1997-09-09T09:09:09+09:09"); + v.push("1997-09-09T09:09:09-09:09"); + v.push("1997-09-09T09:09:09"); + v.push("1997-09-09"); + v.push("09:09:09"); + v.push("1997-09-09T09:09:09.09Z"); + v.push("1997-09-09T09:09:09.09+09:09"); + v.push("1997-09-09T09:09:09.09-09:09"); + v.push("1997-09-09T09:09:09.09"); + v.push("09:09:09.09"); + v +} + #[test] fn test_spanned_field() { #[derive(Deserialize)] @@ -40,4 +58,9 @@ fn test_spanned_field() { "foo = \"1997-09-09T09:09:09Z\"", "\"1997-09-09T09:09:09Z\"" ); + + for expected in good_datetimes() { + let s = format!("foo = {}", expected); + good::(&s, expected); + } } -- cgit v1.2.3 From 0a5fe3fff9a11b9684eb41be302606de0770f22d Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Wed, 9 May 2018 21:26:59 +0200 Subject: Make spanned module private and hide internals --- src/lib.rs | 2 +- src/spanned.rs | 46 +++++++++++++++++++++++++++++++++++++++------ test-suite/tests/spanned.rs | 6 +++--- 3 files changed, 44 insertions(+), 10 deletions(-) (limited to 'test-suite/tests') diff --git a/src/lib.rs b/src/lib.rs index 6fbf5d5..2f038e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -170,6 +170,6 @@ mod tokens; #[doc(hidden)] pub mod macros; -pub mod spanned; +mod spanned; #[doc(no_inline)] pub use spanned::Spanned; diff --git a/src/spanned.rs b/src/spanned.rs index 8fbc529..abbbd49 100644 --- a/src/spanned.rs +++ b/src/spanned.rs @@ -3,7 +3,7 @@ //! extern crate serde_derive; //! //! extern crate toml; -//! use toml::spanned::Spanned; +//! use toml::Spanned; //! //! #[derive(Deserialize)] //! struct Value { @@ -15,8 +15,10 @@ //! //! let u: Value = toml::from_str(t).unwrap(); //! -//! assert_eq!(u.s.start, 4); -//! assert_eq!(u.s.end, 11); +//! assert_eq!(u.s.start(), 4); +//! assert_eq!(u.s.end(), 11); +//! assert_eq!(u.s.get_ref(), "value"); +//! assert_eq!(u.s.into_inner(), String::from("value")); //! } //! ``` @@ -36,11 +38,43 @@ pub const VALUE: &'static str = "$__toml_private_value"; #[derive(Debug)] pub struct Spanned { /// The start range. - pub start: usize, + start: usize, /// The end range (exclusive). - pub end: usize, + end: usize, /// The spanned value. - pub value: T, + value: T, +} + +impl Spanned { + /// Access the start of the span of the contained value. + pub fn start(&self) -> usize { + self.start + } + + /// Access the end of the span of the contained value. + pub fn end(&self) -> usize { + self.end + } + + /// Get the span of the contained value. + pub fn span(&self) -> (usize, usize) { + (self.start, self.end) + } + + /// Consumes the spanned value and returns the contained value. + pub fn into_inner(self) -> T { + self.value + } + + /// Returns a reference to the contained value. + pub fn get_ref(&self) -> &T { + &self.value + } + + /// Returns a mutable reference to the contained value. + pub fn get_mut(&self) -> &T { + &self.value + } } impl<'de, T> de::Deserialize<'de> for Spanned diff --git a/test-suite/tests/spanned.rs b/test-suite/tests/spanned.rs index 4339a0c..1b110f0 100644 --- a/test-suite/tests/spanned.rs +++ b/test-suite/tests/spanned.rs @@ -34,9 +34,9 @@ fn test_spanned_field() { fn good<'de, T>(s: &'de str, expected: &str) where T: serde::Deserialize<'de> { let foo: Foo = toml::from_str(s).unwrap(); - assert_eq!(6, foo.foo.start); - assert_eq!(s.len(), foo.foo.end); - assert_eq!(expected, &s[foo.foo.start..foo.foo.end]); + assert_eq!(6, foo.foo.start()); + assert_eq!(s.len(), foo.foo.end()); + assert_eq!(expected, &s[foo.foo.start()..foo.foo.end()]); } good::("foo = \"foo\"", "\"foo\""); -- cgit v1.2.3