From fd87ce35d563ecdac1f909d4540328365a16661f Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sat, 5 May 2018 14:41:57 -0700 Subject: Proof of concept: reporting span information to a Spanned type --- src/de.rs | 175 ++++++++++++++++++++++++++++++++++++++++----------------- src/lib.rs | 4 ++ src/spanned.rs | 49 ++++++++++++++++ 3 files changed, 177 insertions(+), 51 deletions(-) create mode 100644 src/spanned.rs (limited to 'src') diff --git a/src/de.rs b/src/de.rs index 8677ffc..c80c065 100644 --- a/src/de.rs +++ b/src/de.rs @@ -13,8 +13,9 @@ use std::vec; use serde::de; use serde::de::IntoDeserializer; -use tokens::{Tokenizer, Token, Error as TokenError}; +use tokens::{Tokenizer, Token, Error as TokenError, Span}; use datetime::{SERDE_STRUCT_FIELD_NAME, SERDE_STRUCT_NAME}; +use spanned; /// Deserializes a byte slice into a type. /// @@ -238,7 +239,7 @@ impl<'de, 'b> de::Deserializer<'de> for &'b mut Deserializer<'de> { { if let Some(next) = self.next()? { match next { - Token::String { val, .. } => { + (_, Token::String { val, .. }) => { visitor.visit_enum(val.into_deserializer()) }, _ => Err(Error::from_kind(ErrorKind::ExpectedString)) @@ -504,23 +505,23 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> { fn deserialize_any(self, visitor: V) -> Result where V: de::Visitor<'de>, { - match self.value { - Value::Integer(i) => visitor.visit_i64(i), - Value::Boolean(b) => visitor.visit_bool(b), - Value::Float(f) => visitor.visit_f64(f), - Value::String(Cow::Borrowed(s)) => visitor.visit_borrowed_str(s), - Value::String(Cow::Owned(s)) => visitor.visit_string(s), - Value::Datetime(s) => visitor.visit_map(DatetimeDeserializer { + match self.value.e { + E::Integer(i) => visitor.visit_i64(i), + E::Boolean(b) => visitor.visit_bool(b), + E::Float(f) => visitor.visit_f64(f), + E::String(Cow::Borrowed(s)) => visitor.visit_borrowed_str(s), + E::String(Cow::Owned(s)) => visitor.visit_string(s), + E::Datetime(s) => visitor.visit_map(DatetimeDeserializer { date: s, visited: false, }), - Value::Array(values) => { + E::Array(values) => { let mut s = de::value::SeqDeserializer::new(values.into_iter()); let ret = visitor.visit_seq(&mut s)?; s.end()?; Ok(ret) } - Value::InlineTable(values) => { + E::InlineTable(values) => { visitor.visit_map(InlineTableDeserializer { values: values.into_iter(), next_value: None, @@ -536,7 +537,7 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> { where V: de::Visitor<'de>, { if name == SERDE_STRUCT_NAME && fields == &[SERDE_STRUCT_FIELD_NAME] { - if let Value::Datetime(s) = self.value { + if let E::Datetime(s) = self.value.e { return visitor.visit_map(DatetimeDeserializer { date: s, visited: false, @@ -544,6 +545,16 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> { } } + if name == spanned::NAME && fields == spanned::FIELDS { + let start = self.value.start; + let end = self.value.end; + return visitor.visit_map(SpannedDeserializer { + start: Some(start), + value: Some(self.value), + end: Some(end), + }); + } + self.deserialize_any(visitor) } @@ -563,8 +574,8 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> { ) -> Result where V: de::Visitor<'de> { - match self.value { - Value::String(val) => visitor.visit_enum(val.into_deserializer()), + match self.value.e { + E::String(val) => visitor.visit_enum(val.into_deserializer()), _ => Err(Error::from_kind(ErrorKind::ExpectedString)) } } @@ -594,6 +605,46 @@ impl<'de> de::IntoDeserializer<'de, Error> for Value<'de> { } } +struct SpannedDeserializer<'a> { + start: Option, + value: Option>, + end: Option, +} + +impl<'de> de::MapAccess<'de> for SpannedDeserializer<'de> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result, Error> + where + K: de::DeserializeSeed<'de>, + { + if self.start.is_some() { + seed.deserialize("start".into_deserializer()).map(Some) + } else if self.value.is_some() { + seed.deserialize("value".into_deserializer()).map(Some) + } else if self.end.is_some() { + seed.deserialize("end".into_deserializer()).map(Some) + } else { + Ok(None) + } + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + if let Some(start) = self.start.take() { + seed.deserialize(start.into_deserializer()) + } else if let Some(value) = self.value.take() { + seed.deserialize(value.into_deserializer()) + } else if let Some(end) = self.end.take() { + seed.deserialize(end.into_deserializer()) + } else { + panic!("next_value_seed called before next_key_seed") + } + } +} + struct DatetimeDeserializer<'a> { visited: bool, date: &'a str, @@ -707,7 +758,7 @@ impl<'a> Deserializer<'a> { } match self.peek()? { - Some(Token::LeftBracket) => self.table_header().map(Some), + Some((_, Token::LeftBracket)) => self.table_header().map(Some), Some(_) => self.key_value().map(Some), None => Ok(None), } @@ -725,13 +776,13 @@ impl<'a> Deserializer<'a> { } else { loop { match self.next()? { - Some(Token::RightBracket) => { + Some((_, Token::RightBracket)) => { if array { self.eat(Token::RightBracket)?; } break } - Some(Token::Newline) | + Some((_, Token::Newline)) | None => break, _ => {} } @@ -759,17 +810,27 @@ impl<'a> Deserializer<'a> { fn value(&mut self) -> Result, Error> { let at = self.tokens.current(); let value = match self.next()? { - Some(Token::String { val, .. }) => Value::String(val), - Some(Token::Keylike("true")) => Value::Boolean(true), - Some(Token::Keylike("false")) => Value::Boolean(false), - Some(Token::Keylike(key)) => self.number_or_date(key)?, - Some(Token::Plus) => self.number_leading_plus()?, - Some(Token::LeftBrace) => self.inline_table().map(Value::InlineTable)?, - Some(Token::LeftBracket) => self.array().map(Value::Array)?, + Some((Span { start, end }, Token::String { val, .. })) => { + Value { e: E::String(val), start, end } + } + Some((Span { start, end }, Token::Keylike("true"))) => { + Value { e: E::Boolean(true), start, end } + } + Some((Span { start, end }, Token::Keylike("false"))) => { + Value { e: E::Boolean(false), start, end } + } + Some((_, Token::Keylike(key))) => self.number_or_date(key)?, + Some((_, Token::Plus)) => self.number_leading_plus()?, + Some((Span { start, end }, Token::LeftBrace)) => { + self.inline_table().map(|table| Value { e: E::InlineTable(table), start, end })? + } + Some((Span { start, end }, Token::LeftBracket)) => { + self.array().map(|array| Value { e: E::Array(array), start, end })? + } Some(token) => { return Err(self.error(at, ErrorKind::Wanted { expected: "a value", - found: token.describe(), + found: token.1.describe(), })) } None => return Err(self.eof()), @@ -780,9 +841,11 @@ impl<'a> Deserializer<'a> { fn number_or_date(&mut self, s: &'a str) -> Result, Error> { if s.contains('T') || (s.len() > 1 && s[1..].contains('-')) && !s.contains("e-") { - self.datetime(s, false).map(Value::Datetime) + // FIXME needs span + self.datetime(s, false).map(|d| Value { e: E::Datetime(d), start: 0, end: 0 }) } else if self.eat(Token::Colon)? { - self.datetime(s, true).map(Value::Datetime) + // FIXME needs span + self.datetime(s, true).map(|d| Value { e: E::Datetime(d), start: 0, end: 0 }) } else { self.number(s) } @@ -790,24 +853,27 @@ impl<'a> Deserializer<'a> { fn number(&mut self, s: &'a str) -> Result, Error> { if s.contains('e') || s.contains('E') { - self.float(s, None).map(Value::Float) + // FIXME needs span + self.float(s, None).map(|f| Value { e: E::Float(f), start: 0, end: 0 }) } else if self.eat(Token::Period)? { let at = self.tokens.current(); match self.next()? { - Some(Token::Keylike(after)) => { - self.float(s, Some(after)).map(Value::Float) + Some((_, Token::Keylike(after))) => { + // FIXME needs span + self.float(s, Some(after)).map(|f| Value { e: E::Float(f), start: 0, end: 0 }) } _ => Err(self.error(at, ErrorKind::NumberInvalid)), } } else { - self.integer(s).map(Value::Integer) + // FIXME needs span + self.integer(s).map(|f| Value { e: E::Integer(f), start: 0, end: 0 }) } } fn number_leading_plus(&mut self) -> Result, Error> { let start = self.tokens.current(); match self.next()? { - Some(Token::Keylike(s)) => self.number(s), + Some((_, Token::Keylike(s))) => self.number(s), _ => Err(self.error(start, ErrorKind::NumberInvalid)), } } @@ -884,7 +950,7 @@ impl<'a> Deserializer<'a> { let (a, b) = if suffix.len() == 1 { self.eat(Token::Plus)?; match self.next()? { - Some(Token::Keylike(s)) => { + Some((_, Token::Keylike(s))) => { self.parse_integer(s, false, false)? } _ => return Err(self.error(start, ErrorKind::NumberInvalid)), @@ -927,19 +993,19 @@ impl<'a> Deserializer<'a> { if colon_eaten || self.eat(Token::Colon)? { // minutes match self.next()? { - Some(Token::Keylike(_)) => {} + Some((_, Token::Keylike(_))) => {} _ => return Err(self.error(start, ErrorKind::DateInvalid)), } // Seconds self.expect(Token::Colon)?; match self.next()? { - Some(Token::Keylike(_)) => {} + Some((_, Token::Keylike(_))) => {} _ => return Err(self.error(start, ErrorKind::DateInvalid)), } // Fractional seconds if self.eat(Token::Period)? { match self.next()? { - Some(Token::Keylike(_)) => {} + Some((_, Token::Keylike(_))) => {} _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } @@ -947,13 +1013,13 @@ impl<'a> Deserializer<'a> { // offset if self.eat(Token::Plus)? { match self.next()? { - Some(Token::Keylike(_)) => {} + Some((_, Token::Keylike(_))) => {} _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } if self.eat(Token::Colon)? { match self.next()? { - Some(Token::Keylike(_)) => {} + Some((_, Token::Keylike(_))) => {} _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } @@ -1048,12 +1114,12 @@ impl<'a> Deserializer<'a> { self.tokens.expect(expected).map_err(|e| self.token_error(e)) } - fn next(&mut self) -> Result>, Error> { - self.tokens.next().map(|t| t.map(|t| t.1)).map_err(|e| self.token_error(e)) + fn next(&mut self) -> Result)>, Error> { + self.tokens.next().map_err(|e| self.token_error(e)) } - fn peek(&mut self) -> Result>, Error> { - self.tokens.peek().map(|t| t.map(|t| t.1)).map_err(|e| self.token_error(e)) + fn peek(&mut self) -> Result)>, Error> { + self.tokens.peek().map_err(|e| self.token_error(e)) } fn eof(&self) -> Error { @@ -1300,7 +1366,14 @@ impl<'a> Header<'a> { } #[derive(Debug)] -enum Value<'a> { +struct Value<'a> { + e: E<'a>, + start: usize, + end: usize, +} + +#[derive(Debug)] +enum E<'a> { Integer(i64), Float(f64), Boolean(bool), @@ -1312,14 +1385,14 @@ enum Value<'a> { impl<'a> Value<'a> { fn same_type(&self, other: &Value<'a>) -> bool { - match (self, other) { - (&Value::String(..), &Value::String(..)) | - (&Value::Integer(..), &Value::Integer(..)) | - (&Value::Float(..), &Value::Float(..)) | - (&Value::Boolean(..), &Value::Boolean(..)) | - (&Value::Datetime(..), &Value::Datetime(..)) | - (&Value::Array(..), &Value::Array(..)) | - (&Value::InlineTable(..), &Value::InlineTable(..)) => true, + match (&self.e, &other.e) { + (&E::String(..), &E::String(..)) | + (&E::Integer(..), &E::Integer(..)) | + (&E::Float(..), &E::Float(..)) | + (&E::Boolean(..), &E::Boolean(..)) | + (&E::Datetime(..), &E::Datetime(..)) | + (&E::Array(..), &E::Array(..)) | + (&E::InlineTable(..), &E::InlineTable(..)) => true, _ => false, } diff --git a/src/lib.rs b/src/lib.rs index 1c4842b..cc7bc41 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,6 +153,8 @@ #[macro_use] extern crate serde; +#[macro_use] +extern crate serde_derive; pub mod value; #[doc(no_inline)] @@ -169,3 +171,5 @@ mod tokens; #[doc(hidden)] pub mod macros; + +pub mod spanned; diff --git a/src/spanned.rs b/src/spanned.rs new file mode 100644 index 0000000..04ec9b8 --- /dev/null +++ b/src/spanned.rs @@ -0,0 +1,49 @@ +//! ``` +//! #[macro_use] +//! extern crate serde_derive; +//! +//! extern crate toml; +//! use toml::spanned::Spanned; +//! +//! #[derive(Deserialize)] +//! struct Udoprog { +//! s: Spanned, +//! } +//! +//! fn main() { +//! let t = "s = \"udoprog\"\n"; +//! +//! let u: Udoprog = toml::from_str(t).unwrap(); +//! +//! assert_eq!(u.s.start, 4); +//! assert_eq!(u.s.end, 13); +//! } +//! ``` + +use serde::{Serialize, Serializer}; + +// FIXME: use a more unique name like "toml::Spanned". +#[doc(hidden)] +pub const NAME: &str = "Spanned"; +#[doc(hidden)] +pub const FIELDS: &[&str] = &["value", "start", "end"]; + +/// +#[derive(Deserialize, Debug)] +pub struct Spanned { + /// + pub value: T, + /// + pub start: usize, + /// + pub end: usize, +} + +impl Serialize for Spanned { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.value.serialize(serializer) + } +} -- cgit v1.2.3 From 6ff5c445f37083c785111c904426c94b7a01f4b4 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 03:47:09 +0200 Subject: Use custom struct/field naming to deserialize spans --- src/datetime.rs | 16 +++---- src/de.rs | 76 ++++++++++++++++++------------- src/lib.rs | 1 + src/ser.rs | 6 +-- src/spanned.rs | 136 +++++++++++++++++++++++++++++++++++++++++++++++++------- src/value.rs | 6 +-- 6 files changed, 178 insertions(+), 63 deletions(-) (limited to 'src') diff --git a/src/datetime.rs b/src/datetime.rs index 83b5c0b..c67e2c4 100644 --- a/src/datetime.rs +++ b/src/datetime.rs @@ -40,8 +40,8 @@ pub struct DatetimeParseError { // // In general the TOML encoder/decoder will catch this and not literally emit // these strings but rather emit datetimes as they're intended. -pub const SERDE_STRUCT_FIELD_NAME: &'static str = "$__toml_private_datetime"; -pub const SERDE_STRUCT_NAME: &'static str = "$__toml_private_Datetime"; +pub const FIELD: &'static str = "$__toml_private_datetime"; +pub const NAME: &'static str = "$__toml_private_Datetime"; #[derive(PartialEq, Clone)] struct Date { @@ -311,8 +311,8 @@ impl ser::Serialize for Datetime { { use serde::ser::SerializeStruct; - let mut s = serializer.serialize_struct(SERDE_STRUCT_NAME, 1)?; - s.serialize_field(SERDE_STRUCT_FIELD_NAME, &self.to_string())?; + let mut s = serializer.serialize_struct(NAME, 1)?; + s.serialize_field(FIELD, &self.to_string())?; s.end() } } @@ -343,10 +343,8 @@ impl<'de> de::Deserialize<'de> for Datetime { } } - static FIELDS: [&'static str; 1] = [SERDE_STRUCT_FIELD_NAME]; - deserializer.deserialize_struct(SERDE_STRUCT_NAME, - &FIELDS, - DatetimeVisitor) + static FIELDS: [&'static str; 1] = [FIELD]; + deserializer.deserialize_struct(NAME, &FIELDS, DatetimeVisitor) } } @@ -368,7 +366,7 @@ impl<'de> de::Deserialize<'de> for DatetimeKey { fn visit_str(self, s: &str) -> Result<(), E> where E: de::Error { - if s == SERDE_STRUCT_FIELD_NAME { + if s == FIELD { Ok(()) } else { Err(de::Error::custom("expected field with custom name")) diff --git a/src/de.rs b/src/de.rs index c80c065..cc1e2ca 100644 --- a/src/de.rs +++ b/src/de.rs @@ -14,7 +14,7 @@ use serde::de; use serde::de::IntoDeserializer; use tokens::{Tokenizer, Token, Error as TokenError, Span}; -use datetime::{SERDE_STRUCT_FIELD_NAME, SERDE_STRUCT_NAME}; +use datetime; use spanned; /// Deserializes a byte slice into a type. @@ -536,7 +536,7 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> { visitor: V) -> Result where V: de::Visitor<'de>, { - if name == SERDE_STRUCT_NAME && fields == &[SERDE_STRUCT_FIELD_NAME] { + if name == datetime::NAME && fields == &[datetime::FIELD] { if let E::Datetime(s) = self.value.e { return visitor.visit_map(DatetimeDeserializer { date: s, @@ -545,9 +545,10 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> { } } - if name == spanned::NAME && fields == spanned::FIELDS { + if name == spanned::NAME && fields == &[spanned::START, spanned::END, spanned::VALUE] { let start = self.value.start; let end = self.value.end; + return visitor.visit_map(SpannedDeserializer { start: Some(start), value: Some(self.value), @@ -607,8 +608,8 @@ impl<'de> de::IntoDeserializer<'de, Error> for Value<'de> { struct SpannedDeserializer<'a> { start: Option, - value: Option>, end: Option, + value: Option>, } impl<'de> de::MapAccess<'de> for SpannedDeserializer<'de> { @@ -619,11 +620,11 @@ impl<'de> de::MapAccess<'de> for SpannedDeserializer<'de> { K: de::DeserializeSeed<'de>, { if self.start.is_some() { - seed.deserialize("start".into_deserializer()).map(Some) - } else if self.value.is_some() { - seed.deserialize("value".into_deserializer()).map(Some) + seed.deserialize(spanned::START.into_deserializer()).map(Some) } else if self.end.is_some() { - seed.deserialize("end".into_deserializer()).map(Some) + seed.deserialize(spanned::END.into_deserializer()).map(Some) + } else if self.value.is_some() { + seed.deserialize(spanned::VALUE.into_deserializer()).map(Some) } else { Ok(None) } @@ -635,10 +636,10 @@ impl<'de> de::MapAccess<'de> for SpannedDeserializer<'de> { { if let Some(start) = self.start.take() { seed.deserialize(start.into_deserializer()) + } else if let Some(end) = self.end.take() { + seed.deserialize(end.into_deserializer()) } else if let Some(value) = self.value.take() { seed.deserialize(value.into_deserializer()) - } else if let Some(end) = self.end.take() { - seed.deserialize(end.into_deserializer()) } else { panic!("next_value_seed called before next_key_seed") } @@ -678,7 +679,7 @@ impl<'de> de::Deserializer<'de> for DatetimeFieldDeserializer { fn deserialize_any(self, visitor: V) -> Result where V: de::Visitor<'de>, { - visitor.visit_borrowed_str(SERDE_STRUCT_FIELD_NAME) + visitor.visit_borrowed_str(datetime::FIELD) } forward_to_deserialize_any! { @@ -811,21 +812,25 @@ impl<'a> Deserializer<'a> { let at = self.tokens.current(); let value = match self.next()? { Some((Span { start, end }, Token::String { val, .. })) => { - Value { e: E::String(val), start, end } + Value { e: E::String(val), start: start, end: end } } Some((Span { start, end }, Token::Keylike("true"))) => { - Value { e: E::Boolean(true), start, end } + Value { e: E::Boolean(true), start: start, end: end } } Some((Span { start, end }, Token::Keylike("false"))) => { - Value { e: E::Boolean(false), start, end } + Value { e: E::Boolean(false), start: start, end: end } } - Some((_, Token::Keylike(key))) => self.number_or_date(key)?, + Some((span, Token::Keylike(key))) => self.number_or_date(span, key)?, Some((_, Token::Plus)) => self.number_leading_plus()?, Some((Span { start, end }, Token::LeftBrace)) => { - self.inline_table().map(|table| Value { e: E::InlineTable(table), start, end })? + self.inline_table().map(|table| Value { + e: E::InlineTable(table), + start: start, + end: end + })? } Some((Span { start, end }, Token::LeftBracket)) => { - self.array().map(|array| Value { e: E::Array(array), start, end })? + self.array().map(|array| Value { e: E::Array(array), start: start, end: end })? } Some(token) => { return Err(self.error(at, ErrorKind::Wanted { @@ -838,42 +843,49 @@ impl<'a> Deserializer<'a> { Ok(value) } - fn number_or_date(&mut self, s: &'a str) -> Result, Error> { + fn number_or_date(&mut self, span: Span, s: &'a str) + -> Result, Error> + { if s.contains('T') || (s.len() > 1 && s[1..].contains('-')) && !s.contains("e-") { - // FIXME needs span - self.datetime(s, false).map(|d| Value { e: E::Datetime(d), start: 0, end: 0 }) + self.datetime(s, false).map(|d| Value { + e: E::Datetime(d), + start: span.start, + end: span.end + }) } else if self.eat(Token::Colon)? { - // FIXME needs span - self.datetime(s, true).map(|d| Value { e: E::Datetime(d), start: 0, end: 0 }) + self.datetime(s, true).map(|d| Value { + e: E::Datetime(d), + start: span.start, + end: span.end + }) } else { - self.number(s) + self.number(span, s) } } - fn number(&mut self, s: &'a str) -> Result, Error> { + fn number(&mut self, Span { start, end}: Span, s: &'a str) -> Result, Error> { if s.contains('e') || s.contains('E') { - // FIXME needs span - self.float(s, None).map(|f| Value { e: E::Float(f), start: 0, end: 0 }) + self.float(s, None).map(|f| Value { e: E::Float(f), start: start, end: end }) } else if self.eat(Token::Period)? { let at = self.tokens.current(); match self.next()? { - Some((_, Token::Keylike(after))) => { - // FIXME needs span - self.float(s, Some(after)).map(|f| Value { e: E::Float(f), start: 0, end: 0 }) + Some((Span { start, end }, Token::Keylike(after))) => { + self.float(s, Some(after)).map(|f| Value { + e: E::Float(f), start: start, end: end + }) } _ => Err(self.error(at, ErrorKind::NumberInvalid)), } } else { - // FIXME needs span - self.integer(s).map(|f| Value { e: E::Integer(f), start: 0, end: 0 }) + self.integer(s).map(|f| Value { e: E::Integer(f), start: start, end: end }) } } fn number_leading_plus(&mut self) -> Result, Error> { let start = self.tokens.current(); match self.next()? { - Some((_, Token::Keylike(s))) => self.number(s), + Some((span, Token::Keylike(s))) => self.number(span, s), _ => Err(self.error(start, ErrorKind::NumberInvalid)), } } diff --git a/src/lib.rs b/src/lib.rs index cc7bc41..f42512f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -173,3 +173,4 @@ mod tokens; pub mod macros; pub mod spanned; +pub use spanned::Spanned; diff --git a/src/ser.rs b/src/ser.rs index 58c0b42..b8cec35 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -33,7 +33,7 @@ use std::marker; use std::rc::Rc; use serde::ser; -use datetime::{SERDE_STRUCT_FIELD_NAME, SERDE_STRUCT_NAME}; +use datetime; /// Serialize the given data structure as a TOML byte vector. /// @@ -924,7 +924,7 @@ impl<'a, 'b> ser::Serializer for &'b mut Serializer<'a> { fn serialize_struct(self, name: &'static str, _len: usize) -> Result { - if name == SERDE_STRUCT_NAME { + if name == datetime::NAME { self.array_type("datetime")?; Ok(SerializeTable::Datetime(self)) } else { @@ -1071,7 +1071,7 @@ impl<'a, 'b> ser::SerializeStruct for SerializeTable<'a, 'b> { { match *self { SerializeTable::Datetime(ref mut ser) => { - if key == SERDE_STRUCT_FIELD_NAME { + if key == datetime::FIELD { value.serialize(DateStrEmitter(&mut *ser))?; } else { return Err(Error::DateInvalid) diff --git a/src/spanned.rs b/src/spanned.rs index 04ec9b8..1673d66 100644 --- a/src/spanned.rs +++ b/src/spanned.rs @@ -6,43 +6,147 @@ //! use toml::spanned::Spanned; //! //! #[derive(Deserialize)] -//! struct Udoprog { +//! struct Value { //! s: Spanned, //! } //! //! fn main() { -//! let t = "s = \"udoprog\"\n"; +//! let t = "s = \"value\"\n"; //! -//! let u: Udoprog = toml::from_str(t).unwrap(); +//! let u: Value = toml::from_str(t).unwrap(); //! //! assert_eq!(u.s.start, 4); -//! assert_eq!(u.s.end, 13); +//! assert_eq!(u.s.end, 11); //! } //! ``` -use serde::{Serialize, Serializer}; +use serde::{de, ser}; +use std::fmt; -// FIXME: use a more unique name like "toml::Spanned". #[doc(hidden)] -pub const NAME: &str = "Spanned"; +pub const NAME: &'static str = "$__toml_private_Spanned"; #[doc(hidden)] -pub const FIELDS: &[&str] = &["value", "start", "end"]; +pub const START: &'static str = "$__toml_private_start"; +#[doc(hidden)] +pub const END: &'static str = "$__toml_private_end"; +#[doc(hidden)] +pub const VALUE: &'static str = "$__toml_private_value"; + +macro_rules! key_deserialize { + ($ident:ident, $field:expr, $name:expr) => { + struct $ident; + + impl<'de> de::Deserialize<'de> for $ident { + fn deserialize(deserializer: D) -> Result<$ident, D::Error> + where D: de::Deserializer<'de> + { + struct FieldVisitor; + + impl<'de> de::Visitor<'de> for FieldVisitor { + type Value = (); + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a valid spanned field") + } + + fn visit_str(self, s: &str) -> Result<(), E> + where E: de::Error + { + if s == $field { + Ok(()) + } else { + Err(de::Error::custom( + concat!("expected spanned field `", $name, "`"))) + } + } + } + + deserializer.deserialize_identifier(FieldVisitor)?; + Ok($ident) + } + } + } +} -/// -#[derive(Deserialize, Debug)] + +/// A spanned value, indicating the range at which it is defined in the source. +#[derive(Debug)] pub struct Spanned { - /// - pub value: T, - /// + /// The start range. pub start: usize, - /// + /// The end range (exclusive). pub end: usize, + /// The spanned value. + pub value: T, +} + +impl<'de, T> de::Deserialize<'de> for Spanned + where T: de::Deserialize<'de> +{ + fn deserialize(deserializer: D) -> Result, D::Error> + where D: de::Deserializer<'de> + { + struct SpannedVisitor(::std::marker::PhantomData); + + impl<'de, T> de::Visitor<'de> for SpannedVisitor + where T: de::Deserialize<'de> + { + type Value = Spanned; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a TOML spanned") + } + + fn visit_map(self, mut visitor: V) -> Result, V::Error> + where V: de::MapAccess<'de> + { + let start = visitor.next_key::()?; + + if start.is_none() { + return Err(de::Error::custom("spanned start key not found")) + } + + let start: usize = visitor.next_value()?; + + let end = visitor.next_key::()?; + + if end.is_none() { + return Err(de::Error::custom("spanned end key not found")) + } + + let end: usize = visitor.next_value()?; + + let value = visitor.next_key::()?; + + if value.is_none() { + return Err(de::Error::custom("spanned value key not found")) + } + + let value: T = visitor.next_value()?; + + Ok(Spanned { + start: start, + end: end, + value: value + }) + } + } + + key_deserialize!(StartKey, START, "start"); + key_deserialize!(EndKey, END, "end"); + key_deserialize!(ValueKey, VALUE, "value"); + + let visitor = SpannedVisitor(::std::marker::PhantomData); + + static FIELDS: [&'static str; 3] = [START, END, VALUE]; + deserializer.deserialize_struct(NAME, &FIELDS, visitor) + } } -impl Serialize for Spanned { +impl ser::Serialize for Spanned { fn serialize(&self, serializer: S) -> Result where - S: Serializer, + S: ser::Serializer, { self.value.serialize(serializer) } diff --git a/src/value.rs b/src/value.rs index 54fa45c..21a8205 100644 --- a/src/value.rs +++ b/src/value.rs @@ -12,7 +12,7 @@ use serde::de; use serde::de::IntoDeserializer; pub use datetime::{Datetime, DatetimeParseError}; -use datetime::{DatetimeFromString, SERDE_STRUCT_FIELD_NAME}; +use datetime::{self, DatetimeFromString}; /// Representation of a TOML value. #[derive(PartialEq, Clone, Debug)] @@ -925,7 +925,7 @@ impl<'a, 'de> de::Visitor<'de> for DatetimeOrTable<'a> { fn visit_str(self, s: &str) -> Result where E: de::Error, { - if s == SERDE_STRUCT_FIELD_NAME { + if s == datetime::FIELD { Ok(true) } else { self.key.push_str(s); @@ -936,7 +936,7 @@ impl<'a, 'de> de::Visitor<'de> for DatetimeOrTable<'a> { fn visit_string(self, s: String) -> Result where E: de::Error, { - if s == SERDE_STRUCT_FIELD_NAME { + if s == datetime::FIELD { Ok(true) } else { *self.key = s; -- cgit v1.2.3 From 6c067202e061910d5018c4f96e9a7595c4291a08 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 04:05:05 +0200 Subject: Adjust spans for more complex types --- src/de.rs | 52 +++++++++++++++++++++++++++++++++------------------- src/tokens.rs | 29 +++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 27 deletions(-) (limited to 'src') diff --git a/src/de.rs b/src/de.rs index cc1e2ca..4e2b4b7 100644 --- a/src/de.rs +++ b/src/de.rs @@ -821,16 +821,20 @@ impl<'a> Deserializer<'a> { Value { e: E::Boolean(false), start: start, end: end } } Some((span, Token::Keylike(key))) => self.number_or_date(span, key)?, - Some((_, Token::Plus)) => self.number_leading_plus()?, - Some((Span { start, end }, Token::LeftBrace)) => { - self.inline_table().map(|table| Value { + Some((span, Token::Plus)) => self.number_leading_plus(span)?, + Some((Span { start, .. }, Token::LeftBrace)) => { + self.inline_table().map(|(Span { end, .. }, table)| Value { e: E::InlineTable(table), start: start, end: end })? } - Some((Span { start, end }, Token::LeftBracket)) => { - self.array().map(|array| Value { e: E::Array(array), start: start, end: end })? + Some((Span { start, .. }, Token::LeftBracket)) => { + self.array().map(|(Span { end, .. }, array)| Value { + e: E::Array(array), + start: start, + end: end + })? } Some(token) => { return Err(self.error(at, ErrorKind::Wanted { @@ -882,11 +886,13 @@ impl<'a> Deserializer<'a> { } } - fn number_leading_plus(&mut self) -> Result, Error> { - let start = self.tokens.current(); + fn number_leading_plus(&mut self, Span { start, .. }: Span) -> Result, Error> { + let start_token = self.tokens.current(); match self.next()? { - Some((span, Token::Keylike(s))) => self.number(span, s), - _ => Err(self.error(start, ErrorKind::NumberInvalid)), + Some((Span { end, .. }, Token::Keylike(s))) => { + self.number(Span { start: start, end: end }, s) + }, + _ => Err(self.error(start_token, ErrorKind::NumberInvalid)), } } @@ -1042,11 +1048,11 @@ impl<'a> Deserializer<'a> { // TODO(#140): shouldn't buffer up this entire table in memory, it'd be // great to defer parsing everything until later. - fn inline_table(&mut self) -> Result, Value<'a>)>, Error> { + fn inline_table(&mut self) -> Result<(Span, Vec<(Cow<'a, str>, Value<'a>)>), Error> { let mut ret = Vec::new(); self.eat_whitespace()?; - if self.eat(Token::RightBrace)? { - return Ok(ret) + if let Some(span) = self.eat_spanned(Token::RightBrace)? { + return Ok((span, ret)) } loop { let key = self.table_key()?; @@ -1056,8 +1062,8 @@ impl<'a> Deserializer<'a> { ret.push((key, self.value()?)); self.eat_whitespace()?; - if self.eat(Token::RightBrace)? { - return Ok(ret) + if let Some(span) = self.eat_spanned(Token::RightBrace)? { + return Ok((span, ret)) } self.expect(Token::Comma)?; self.eat_whitespace()?; @@ -1066,7 +1072,7 @@ impl<'a> Deserializer<'a> { // TODO(#140): shouldn't buffer up this entire array in memory, it'd be // great to defer parsing everything until later. - fn array(&mut self) -> Result>, Error> { + fn array(&mut self) -> Result<(Span, Vec>), Error> { let mut ret = Vec::new(); let intermediate = |me: &mut Deserializer| { @@ -1081,8 +1087,8 @@ impl<'a> Deserializer<'a> { loop { intermediate(self)?; - if self.eat(Token::RightBracket)? { - return Ok(ret) + if let Some(span) = self.eat_spanned(Token::RightBracket)? { + return Ok((span, ret)) } let at = self.tokens.current(); let value = self.value()?; @@ -1098,8 +1104,8 @@ impl<'a> Deserializer<'a> { } } intermediate(self)?; - self.expect(Token::RightBracket)?; - Ok(ret) + let span = self.expect_spanned(Token::RightBracket)?; + Ok((span, ret)) } fn table_key(&mut self) -> Result, Error> { @@ -1122,10 +1128,18 @@ impl<'a> Deserializer<'a> { self.tokens.eat(expected).map_err(|e| self.token_error(e)) } + fn eat_spanned(&mut self, expected: Token<'a>) -> Result, Error> { + self.tokens.eat_spanned(expected).map_err(|e| self.token_error(e)) + } + fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> { self.tokens.expect(expected).map_err(|e| self.token_error(e)) } + fn expect_spanned(&mut self, expected: Token<'a>) -> Result { + self.tokens.expect_spanned(expected).map_err(|e| self.token_error(e)) + } + fn next(&mut self) -> Result)>, Error> { self.tokens.next().map_err(|e| self.token_error(e)) } diff --git a/src/tokens.rs b/src/tokens.rs index bcabd94..bfdc74e 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -118,21 +118,34 @@ impl<'a> Tokenizer<'a> { } pub fn eat(&mut self, expected: Token<'a>) -> Result { - match self.peek()? { - Some((_, ref found)) if expected == *found => {} - Some(_) => return Ok(false), - None => return Ok(false), - } + self.eat_spanned(expected).map(|s| s.is_some()) + } + + /// Eat a value, returning it's span if it was consumed. + pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result, Error> { + let span = match self.peek()? { + Some((span, ref found)) if expected == *found => span, + Some(_) => return Ok(None), + None => return Ok(None), + }; + drop(self.next()); - Ok(true) + Ok(Some(span)) } pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> { + // ignore span + let _ = self.expect_spanned(expected)?; + Ok(()) + } + + /// Expect the given token returning its span. + pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result { let current = self.current(); match self.next()? { - Some((_, found)) => { + Some((span, found)) => { if expected == found { - Ok(()) + Ok(span) } else { Err(Error::Wanted { at: current, -- cgit v1.2.3 From 579cf491e6cca04b5e4cfe63f3132eed1f2a4e93 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 04:56:25 +0200 Subject: Use BorrowedStrDeserializer instead of generated key deserializers --- src/de.rs | 7 ++++--- src/lib.rs | 2 -- src/spanned.rs | 53 +++-------------------------------------------------- 3 files changed, 7 insertions(+), 55 deletions(-) (limited to 'src') diff --git a/src/de.rs b/src/de.rs index 4e2b4b7..5521e00 100644 --- a/src/de.rs +++ b/src/de.rs @@ -12,6 +12,7 @@ use std::vec; use serde::de; use serde::de::IntoDeserializer; +use serde::de::value::BorrowedStrDeserializer; use tokens::{Tokenizer, Token, Error as TokenError, Span}; use datetime; @@ -620,11 +621,11 @@ impl<'de> de::MapAccess<'de> for SpannedDeserializer<'de> { K: de::DeserializeSeed<'de>, { if self.start.is_some() { - seed.deserialize(spanned::START.into_deserializer()).map(Some) + seed.deserialize(BorrowedStrDeserializer::new(spanned::START)).map(Some) } else if self.end.is_some() { - seed.deserialize(spanned::END.into_deserializer()).map(Some) + seed.deserialize(BorrowedStrDeserializer::new(spanned::END)).map(Some) } else if self.value.is_some() { - seed.deserialize(spanned::VALUE.into_deserializer()).map(Some) + seed.deserialize(BorrowedStrDeserializer::new(spanned::VALUE)).map(Some) } else { Ok(None) } diff --git a/src/lib.rs b/src/lib.rs index f42512f..b90612c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,8 +153,6 @@ #[macro_use] extern crate serde; -#[macro_use] -extern crate serde_derive; pub mod value; #[doc(no_inline)] diff --git a/src/spanned.rs b/src/spanned.rs index 1673d66..8fbc529 100644 --- a/src/spanned.rs +++ b/src/spanned.rs @@ -32,43 +32,6 @@ pub const END: &'static str = "$__toml_private_end"; #[doc(hidden)] pub const VALUE: &'static str = "$__toml_private_value"; -macro_rules! key_deserialize { - ($ident:ident, $field:expr, $name:expr) => { - struct $ident; - - impl<'de> de::Deserialize<'de> for $ident { - fn deserialize(deserializer: D) -> Result<$ident, D::Error> - where D: de::Deserializer<'de> - { - struct FieldVisitor; - - impl<'de> de::Visitor<'de> for FieldVisitor { - type Value = (); - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a valid spanned field") - } - - fn visit_str(self, s: &str) -> Result<(), E> - where E: de::Error - { - if s == $field { - Ok(()) - } else { - Err(de::Error::custom( - concat!("expected spanned field `", $name, "`"))) - } - } - } - - deserializer.deserialize_identifier(FieldVisitor)?; - Ok($ident) - } - } - } -} - - /// A spanned value, indicating the range at which it is defined in the source. #[derive(Debug)] pub struct Spanned { @@ -100,25 +63,19 @@ impl<'de, T> de::Deserialize<'de> for Spanned fn visit_map(self, mut visitor: V) -> Result, V::Error> where V: de::MapAccess<'de> { - let start = visitor.next_key::()?; - - if start.is_none() { + if visitor.next_key()? != Some(START) { return Err(de::Error::custom("spanned start key not found")) } let start: usize = visitor.next_value()?; - let end = visitor.next_key::()?; - - if end.is_none() { + if visitor.next_key()? != Some(END) { return Err(de::Error::custom("spanned end key not found")) } let end: usize = visitor.next_value()?; - let value = visitor.next_key::()?; - - if value.is_none() { + if visitor.next_key()? != Some(VALUE) { return Err(de::Error::custom("spanned value key not found")) } @@ -132,10 +89,6 @@ impl<'de, T> de::Deserialize<'de> for Spanned } } - key_deserialize!(StartKey, START, "start"); - key_deserialize!(EndKey, END, "end"); - key_deserialize!(ValueKey, VALUE, "value"); - let visitor = SpannedVisitor(::std::marker::PhantomData); static FIELDS: [&'static str; 3] = [START, END, VALUE]; -- cgit v1.2.3 From 2b40d542a0ea645c5e3c132aabc0082dffa2f554 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 7 May 2018 05:36:09 +0200 Subject: Fix incorrectly reported spans for Datetime --- src/de.rs | 36 +++++++++++++++++++++++------------- src/lib.rs | 1 + 2 files changed, 24 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/de.rs b/src/de.rs index 5521e00..1d43cc9 100644 --- a/src/de.rs +++ b/src/de.rs @@ -853,16 +853,16 @@ impl<'a> Deserializer<'a> { { if s.contains('T') || (s.len() > 1 && s[1..].contains('-')) && !s.contains("e-") { - self.datetime(s, false).map(|d| Value { + self.datetime(span, s, false).map(|(Span { start, end }, d)| Value { e: E::Datetime(d), - start: span.start, - end: span.end + start: start, + end: end }) } else if self.eat(Token::Colon)? { - self.datetime(s, true).map(|d| Value { + self.datetime(span, s, true).map(|(Span { start, end }, d)| Value { e: E::Datetime(d), - start: span.start, - end: span.end + start: start, + end: end }) } else { self.number(span, s) @@ -1006,9 +1006,10 @@ impl<'a> Deserializer<'a> { }) } - fn datetime(&mut self, date: &'a str, colon_eaten: bool) - -> Result<&'a str, Error> { + fn datetime(&mut self, mut span: Span, date: &'a str, colon_eaten: bool) + -> Result<(Span, &'a str), Error> { let start = self.tokens.substr_offset(date); + if colon_eaten || self.eat(Token::Colon)? { // minutes match self.next()? { @@ -1018,13 +1019,17 @@ impl<'a> Deserializer<'a> { // Seconds self.expect(Token::Colon)?; match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } // Fractional seconds if self.eat(Token::Period)? { match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } @@ -1032,19 +1037,24 @@ impl<'a> Deserializer<'a> { // offset if self.eat(Token::Plus)? { match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } if self.eat(Token::Colon)? { match self.next()? { - Some((_, Token::Keylike(_))) => {} + Some((Span { end, .. }, Token::Keylike(_))) => { + span.end = end; + }, _ => return Err(self.error(start, ErrorKind::DateInvalid)), } } } + let end = self.tokens.current(); - Ok(&self.tokens.input()[start..end]) + Ok((span, &self.tokens.input()[start..end])) } // TODO(#140): shouldn't buffer up this entire table in memory, it'd be diff --git a/src/lib.rs b/src/lib.rs index b90612c..6fbf5d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -171,4 +171,5 @@ mod tokens; pub mod macros; pub mod spanned; +#[doc(no_inline)] pub use spanned::Spanned; -- cgit v1.2.3 From 0a5fe3fff9a11b9684eb41be302606de0770f22d Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Wed, 9 May 2018 21:26:59 +0200 Subject: Make spanned module private and hide internals --- src/lib.rs | 2 +- src/spanned.rs | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/lib.rs b/src/lib.rs index 6fbf5d5..2f038e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -170,6 +170,6 @@ mod tokens; #[doc(hidden)] pub mod macros; -pub mod spanned; +mod spanned; #[doc(no_inline)] pub use spanned::Spanned; diff --git a/src/spanned.rs b/src/spanned.rs index 8fbc529..abbbd49 100644 --- a/src/spanned.rs +++ b/src/spanned.rs @@ -3,7 +3,7 @@ //! extern crate serde_derive; //! //! extern crate toml; -//! use toml::spanned::Spanned; +//! use toml::Spanned; //! //! #[derive(Deserialize)] //! struct Value { @@ -15,8 +15,10 @@ //! //! let u: Value = toml::from_str(t).unwrap(); //! -//! assert_eq!(u.s.start, 4); -//! assert_eq!(u.s.end, 11); +//! assert_eq!(u.s.start(), 4); +//! assert_eq!(u.s.end(), 11); +//! assert_eq!(u.s.get_ref(), "value"); +//! assert_eq!(u.s.into_inner(), String::from("value")); //! } //! ``` @@ -36,11 +38,43 @@ pub const VALUE: &'static str = "$__toml_private_value"; #[derive(Debug)] pub struct Spanned { /// The start range. - pub start: usize, + start: usize, /// The end range (exclusive). - pub end: usize, + end: usize, /// The spanned value. - pub value: T, + value: T, +} + +impl Spanned { + /// Access the start of the span of the contained value. + pub fn start(&self) -> usize { + self.start + } + + /// Access the end of the span of the contained value. + pub fn end(&self) -> usize { + self.end + } + + /// Get the span of the contained value. + pub fn span(&self) -> (usize, usize) { + (self.start, self.end) + } + + /// Consumes the spanned value and returns the contained value. + pub fn into_inner(self) -> T { + self.value + } + + /// Returns a reference to the contained value. + pub fn get_ref(&self) -> &T { + &self.value + } + + /// Returns a mutable reference to the contained value. + pub fn get_mut(&self) -> &T { + &self.value + } } impl<'de, T> de::Deserialize<'de> for Spanned -- cgit v1.2.3