Merge pull request #239 from udoprog/spans

Support spans when deserializing serde structures

7 files changed, 361 insertions, 96 deletions

diff --git a/src/datetime.rs b/src/datetime.rs
index 83b5c0b..c67e2c4 100644
--- a/src/datetime.rs
+++ b/src/datetime.rs
@@ -40,8 +40,8 @@ pub struct DatetimeParseError {
 //
 // In general the TOML encoder/decoder will catch this and not literally emit
 // these strings but rather emit datetimes as they're intended.
-pub const SERDE_STRUCT_FIELD_NAME: &'static str = "$__toml_private_datetime";
-pub const SERDE_STRUCT_NAME: &'static str = "$__toml_private_Datetime";
+pub const FIELD: &'static str = "$__toml_private_datetime";
+pub const NAME: &'static str = "$__toml_private_Datetime";
 
 #[derive(PartialEq, Clone)]
 struct Date {
@@ -311,8 +311,8 @@ impl ser::Serialize for Datetime {
     {
         use serde::ser::SerializeStruct;
 
-        let mut s = serializer.serialize_struct(SERDE_STRUCT_NAME, 1)?;
-        s.serialize_field(SERDE_STRUCT_FIELD_NAME, &self.to_string())?;
+        let mut s = serializer.serialize_struct(NAME, 1)?;
+        s.serialize_field(FIELD, &self.to_string())?;
         s.end()
     }
 }
@@ -343,10 +343,8 @@ impl<'de> de::Deserialize<'de> for Datetime {
             }
         }
 
-        static FIELDS: [&'static str; 1] = [SERDE_STRUCT_FIELD_NAME];
-        deserializer.deserialize_struct(SERDE_STRUCT_NAME,
-                                        &FIELDS,
-                                        DatetimeVisitor)
+        static FIELDS: [&'static str; 1] = [FIELD];
+        deserializer.deserialize_struct(NAME, &FIELDS, DatetimeVisitor)
     }
 }
 
@@ -368,7 +366,7 @@ impl<'de> de::Deserialize<'de> for DatetimeKey {
             fn visit_str<E>(self, s: &str) -> Result<(), E>
                 where E: de::Error
             {
-                if s == SERDE_STRUCT_FIELD_NAME {
+                if s == FIELD {
                     Ok(())
                 } else {
                     Err(de::Error::custom("expected field with custom name"))
diff --git a/src/de.rs b/src/de.rs
index 8677ffc..1d43cc9 100644
--- a/src/de.rs
+++ b/src/de.rs
@@ -12,9 +12,11 @@ use std::vec;
 
 use serde::de;
 use serde::de::IntoDeserializer;
+use serde::de::value::BorrowedStrDeserializer;
 
-use tokens::{Tokenizer, Token, Error as TokenError};
-use datetime::{SERDE_STRUCT_FIELD_NAME, SERDE_STRUCT_NAME};
+use tokens::{Tokenizer, Token, Error as TokenError, Span};
+use datetime;
+use spanned;
 
 /// Deserializes a byte slice into a type.
 ///
@@ -238,7 +240,7 @@ impl<'de, 'b> de::Deserializer<'de> for &'b mut Deserializer<'de> {
     {
         if let Some(next) = self.next()? {
             match next {
-                Token::String { val, .. } => {
+                (_, Token::String { val, .. }) => {
                     visitor.visit_enum(val.into_deserializer())
                 },
                 _ => Err(Error::from_kind(ErrorKind::ExpectedString))
@@ -504,23 +506,23 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> {
     fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Error>
         where V: de::Visitor<'de>,
     {
-        match self.value {
-            Value::Integer(i) => visitor.visit_i64(i),
-            Value::Boolean(b) => visitor.visit_bool(b),
-            Value::Float(f) => visitor.visit_f64(f),
-            Value::String(Cow::Borrowed(s)) => visitor.visit_borrowed_str(s),
-            Value::String(Cow::Owned(s)) => visitor.visit_string(s),
-            Value::Datetime(s) => visitor.visit_map(DatetimeDeserializer {
+        match self.value.e {
+            E::Integer(i) => visitor.visit_i64(i),
+            E::Boolean(b) => visitor.visit_bool(b),
+            E::Float(f) => visitor.visit_f64(f),
+            E::String(Cow::Borrowed(s)) => visitor.visit_borrowed_str(s),
+            E::String(Cow::Owned(s)) => visitor.visit_string(s),
+            E::Datetime(s) => visitor.visit_map(DatetimeDeserializer {
                 date: s,
                 visited: false,
             }),
-            Value::Array(values) => {
+            E::Array(values) => {
                 let mut s = de::value::SeqDeserializer::new(values.into_iter());
                 let ret = visitor.visit_seq(&mut s)?;
                 s.end()?;
                 Ok(ret)
             }
-            Value::InlineTable(values) => {
+            E::InlineTable(values) => {
                 visitor.visit_map(InlineTableDeserializer {
                     values: values.into_iter(),
                     next_value: None,
@@ -535,8 +537,8 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> {
                              visitor: V) -> Result<V::Value, Error>
         where V: de::Visitor<'de>,
     {
-        if name == SERDE_STRUCT_NAME && fields == &[SERDE_STRUCT_FIELD_NAME] {
-            if let Value::Datetime(s) = self.value {
+        if name == datetime::NAME && fields == &[datetime::FIELD] {
+            if let E::Datetime(s) = self.value.e {
                 return visitor.visit_map(DatetimeDeserializer {
                     date: s,
                     visited: false,
@@ -544,6 +546,17 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> {
             }
         }
 
+        if name == spanned::NAME && fields == &[spanned::START, spanned::END, spanned::VALUE] {
+            let start = self.value.start;
+            let end = self.value.end;
+
+            return visitor.visit_map(SpannedDeserializer {
+                start: Some(start),
+                value: Some(self.value),
+                end: Some(end),
+            });
+        }
+
         self.deserialize_any(visitor)
     }
 
@@ -563,8 +576,8 @@ impl<'de> de::Deserializer<'de> for ValueDeserializer<'de> {
     ) -> Result<V::Value, Error>
         where V: de::Visitor<'de>
     {
-        match self.value {
-            Value::String(val) => visitor.visit_enum(val.into_deserializer()),
+        match self.value.e {
+            E::String(val) => visitor.visit_enum(val.into_deserializer()),
             _ => Err(Error::from_kind(ErrorKind::ExpectedString))
         }
     }
@@ -594,6 +607,46 @@ impl<'de> de::IntoDeserializer<'de, Error> for Value<'de> {
     }
 }
 
+struct SpannedDeserializer<'a> {
+    start: Option<usize>,
+    end: Option<usize>,
+    value: Option<Value<'a>>,
+}
+
+impl<'de> de::MapAccess<'de> for SpannedDeserializer<'de> {
+    type Error = Error;
+
+    fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Error>
+    where
+        K: de::DeserializeSeed<'de>,
+    {
+        if self.start.is_some() {
+            seed.deserialize(BorrowedStrDeserializer::new(spanned::START)).map(Some)
+        } else if self.end.is_some() {
+            seed.deserialize(BorrowedStrDeserializer::new(spanned::END)).map(Some)
+        } else if self.value.is_some() {
+            seed.deserialize(BorrowedStrDeserializer::new(spanned::VALUE)).map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Error>
+    where
+        V: de::DeserializeSeed<'de>,
+    {
+        if let Some(start) = self.start.take() {
+            seed.deserialize(start.into_deserializer())
+        } else if  let Some(end) = self.end.take() {
+            seed.deserialize(end.into_deserializer())
+        } else if let Some(value) = self.value.take() {
+            seed.deserialize(value.into_deserializer())
+        } else {
+            panic!("next_value_seed called before next_key_seed")
+        }
+    }
+}
+
 struct DatetimeDeserializer<'a> {
     visited: bool,
     date: &'a str,
@@ -627,7 +680,7 @@ impl<'de> de::Deserializer<'de> for DatetimeFieldDeserializer {
     fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Error>
         where V: de::Visitor<'de>,
     {
-        visitor.visit_borrowed_str(SERDE_STRUCT_FIELD_NAME)
+        visitor.visit_borrowed_str(datetime::FIELD)
     }
 
     forward_to_deserialize_any! {
@@ -707,7 +760,7 @@ impl<'a> Deserializer<'a> {
         }
 
         match self.peek()? {
-            Some(Token::LeftBracket) => self.table_header().map(Some),
+            Some((_, Token::LeftBracket)) => self.table_header().map(Some),
             Some(_) => self.key_value().map(Some),
             None => Ok(None),
         }
@@ -725,13 +778,13 @@ impl<'a> Deserializer<'a> {
         } else {
             loop {
                 match self.next()? {
-                    Some(Token::RightBracket) => {
+                    Some((_, Token::RightBracket)) => {
                         if array {
                             self.eat(Token::RightBracket)?;
                         }
                         break
                     }
-                    Some(Token::Newline) |
+                    Some((_, Token::Newline)) |
                     None => break,
                     _ => {}
                 }
@@ -759,17 +812,35 @@ impl<'a> Deserializer<'a> {
     fn value(&mut self) -> Result<Value<'a>, Error> {
         let at = self.tokens.current();
         let value = match self.next()? {
-            Some(Token::String { val, .. }) => Value::String(val),
-            Some(Token::Keylike("true")) => Value::Boolean(true),
-            Some(Token::Keylike("false")) => Value::Boolean(false),
-            Some(Token::Keylike(key)) => self.number_or_date(key)?,
-            Some(Token::Plus) => self.number_leading_plus()?,
-            Some(Token::LeftBrace) => self.inline_table().map(Value::InlineTable)?,
-            Some(Token::LeftBracket) => self.array().map(Value::Array)?,
+            Some((Span { start, end }, Token::String { val, .. })) => {
+                Value { e: E::String(val), start: start, end: end }
+            }
+            Some((Span { start, end }, Token::Keylike("true"))) => {
+                Value { e: E::Boolean(true), start: start, end: end }
+            }
+            Some((Span { start, end }, Token::Keylike("false"))) => {
+                Value { e: E::Boolean(false), start: start, end: end }
+            }
+            Some((span, Token::Keylike(key))) => self.number_or_date(span, key)?,
+            Some((span, Token::Plus)) => self.number_leading_plus(span)?,
+            Some((Span { start, .. }, Token::LeftBrace)) => {
+                self.inline_table().map(|(Span { end, .. }, table)| Value {
+                    e: E::InlineTable(table),
+                    start: start,
+                    end: end
+                })?
+            }
+            Some((Span { start, .. }, Token::LeftBracket)) => {
+                self.array().map(|(Span { end, .. }, array)| Value {
+                    e: E::Array(array),
+                    start: start,
+                    end: end
+                })?
+            }
             Some(token) => {
                 return Err(self.error(at, ErrorKind::Wanted {
                     expected: "a value",
-                    found: token.describe(),
+                    found: token.1.describe(),
                 }))
             }
             None => return Err(self.eof()),
@@ -777,38 +848,52 @@ impl<'a> Deserializer<'a> {
         Ok(value)
     }
 
-    fn number_or_date(&mut self, s: &'a str) -> Result<Value<'a>, Error> {
+    fn number_or_date(&mut self, span: Span, s: &'a str)
+        -> Result<Value<'a>, Error>
+    {
         if s.contains('T') || (s.len() > 1 && s[1..].contains('-')) &&
            !s.contains("e-") {
-            self.datetime(s, false).map(Value::Datetime)
+            self.datetime(span, s, false).map(|(Span { start, end }, d)| Value {
+                e: E::Datetime(d),
+                start: start,
+                end: end
+            })
         } else if self.eat(Token::Colon)? {
-            self.datetime(s, true).map(Value::Datetime)
+            self.datetime(span, s, true).map(|(Span { start, end }, d)| Value {
+                e: E::Datetime(d),
+                start: start,
+                end: end
+            })
         } else {
-            self.number(s)
+            self.number(span, s)
         }
     }
 
-    fn number(&mut self, s: &'a str) -> Result<Value<'a>, Error> {
+    fn number(&mut self, Span { start, end}: Span, s: &'a str) -> Result<Value<'a>, Error> {
         if s.contains('e') || s.contains('E') {
-            self.float(s, None).map(Value::Float)
+            self.float(s, None).map(|f| Value { e: E::Float(f), start: start, end: end })
         } else if self.eat(Token::Period)? {
             let at = self.tokens.current();
             match self.next()? {
-                Some(Token::Keylike(after)) => {
-                    self.float(s, Some(after)).map(Value::Float)
+                Some((Span { start, end }, Token::Keylike(after))) => {
+                    self.float(s, Some(after)).map(|f| Value {
+                        e: E::Float(f), start: start, end: end
+                    })
                 }
                 _ => Err(self.error(at, ErrorKind::NumberInvalid)),
             }
         } else {
-            self.integer(s).map(Value::Integer)
+            self.integer(s).map(|f| Value { e: E::Integer(f), start: start, end: end })
         }
     }
 
-    fn number_leading_plus(&mut self) -> Result<Value<'a>, Error> {
-        let start = self.tokens.current();
+    fn number_leading_plus(&mut self, Span { start, .. }: Span) -> Result<Value<'a>, Error> {
+        let start_token = self.tokens.current();
         match self.next()? {
-            Some(Token::Keylike(s)) => self.number(s),
-            _ => Err(self.error(start, ErrorKind::NumberInvalid)),
+            Some((Span { end, .. }, Token::Keylike(s))) => {
+                self.number(Span { start: start, end: end }, s)
+            },
+            _ => Err(self.error(start_token, ErrorKind::NumberInvalid)),
         }
     }
 
@@ -884,7 +969,7 @@ impl<'a> Deserializer<'a> {
             let (a, b) = if suffix.len() == 1 {
                 self.eat(Token::Plus)?;
                 match self.next()? {
-                    Some(Token::Keylike(s)) => {
+                    Some((_, Token::Keylike(s))) => {
                         self.parse_integer(s, false, false)?
                     }
                     _ => return Err(self.error(start, ErrorKind::NumberInvalid)),
@@ -921,25 +1006,30 @@ impl<'a> Deserializer<'a> {
         })
     }
 
-    fn datetime(&mut self, date: &'a str, colon_eaten: bool)
-                -> Result<&'a str, Error> {
+    fn datetime(&mut self, mut span: Span, date: &'a str, colon_eaten: bool)
+                -> Result<(Span, &'a str), Error> {
         let start = self.tokens.substr_offset(date);
+
         if colon_eaten || self.eat(Token::Colon)? {
             // minutes
             match self.next()? {
-                Some(Token::Keylike(_)) => {}
+                Some((_, Token::Keylike(_))) => {}
                 _ => return Err(self.error(start, ErrorKind::DateInvalid)),
             }
             // Seconds
             self.expect(Token::Colon)?;
             match self.next()? {
-                Some(Token::Keylike(_)) => {}
+                Some((Span { end, .. }, Token::Keylike(_))) => {
+                    span.end = end;
+                },
                 _ => return Err(self.error(start, ErrorKind::DateInvalid)),
             }
             // Fractional seconds
             if self.eat(Token::Period)? {
                 match self.next()? {
-                    Some(Token::Keylike(_)) => {}
+                    Some((Span { end, .. }, Token::Keylike(_))) => {
+                        span.end = end;
+                    },
                     _ => return Err(self.error(start, ErrorKind::DateInvalid)),
                 }
             }
@@ -947,28 +1037,33 @@ impl<'a> Deserializer<'a> {
             // offset
             if self.eat(Token::Plus)? {
                 match self.next()? {
-                    Some(Token::Keylike(_)) => {}
+                    Some((Span { end, .. }, Token::Keylike(_))) => {
+                        span.end = end;
+                    },
                     _ => return Err(self.error(start, ErrorKind::DateInvalid)),
                 }
             }
             if self.eat(Token::Colon)? {
                 match self.next()? {
-                    Some(Token::Keylike(_)) => {}
+                    Some((Span { end, .. }, Token::Keylike(_))) => {
+                        span.end = end;
+                    },
                     _ => return Err(self.error(start, ErrorKind::DateInvalid)),
                 }
             }
         }
+
         let end = self.tokens.current();
-        Ok(&self.tokens.input()[start..end])
+        Ok((span, &self.tokens.input()[start..end]))
     }
 
     // TODO(#140): shouldn't buffer up this entire table in memory, it'd be
     // great to defer parsing everything until later.
-    fn inline_table(&mut self) -> Result<Vec<(Cow<'a, str>, Value<'a>)>, Error> {
+    fn inline_table(&mut self) -> Result<(Span, Vec<(Cow<'a, str>, Value<'a>)>), Error> {
         let mut ret = Vec::new();
         self.eat_whitespace()?;
-        if self.eat(Token::RightBrace)? {
-            return Ok(ret)
+        if let Some(span) = self.eat_spanned(Token::RightBrace)? {
+            return Ok((span, ret))
         }
         loop {
             let key = self.table_key()?;
@@ -978,8 +1073,8 @@ impl<'a> Deserializer<'a> {
             ret.push((key, self.value()?));
 
             self.eat_whitespace()?;
-            if self.eat(Token::RightBrace)? {
-                return Ok(ret)
+            if let Some(span) = self.eat_spanned(Token::RightBrace)? {
+                return Ok((span, ret))
             }
             self.expect(Token::Comma)?;
             self.eat_whitespace()?;
@@ -988,7 +1083,7 @@ impl<'a> Deserializer<'a> {
 
     // TODO(#140): shouldn't buffer up this entire array in memory, it'd be
     // great to defer parsing everything until later.
-    fn array(&mut self) -> Result<Vec<Value<'a>>, Error> {
+    fn array(&mut self) -> Result<(Span, Vec<Value<'a>>), Error> {
         let mut ret = Vec::new();
 
         let intermediate = |me: &mut Deserializer| {
@@ -1003,8 +1098,8 @@ impl<'a> Deserializer<'a> {
 
         loop {
             intermediate(self)?;
-            if self.eat(Token::RightBracket)? {
-                return Ok(ret)
+            if let Some(span) = self.eat_spanned(Token::RightBracket)? {
+                return Ok((span, ret))
             }
             let at = self.tokens.current();
             let value = self.value()?;
@@ -1020,8 +1115,8 @@ impl<'a> Deserializer<'a> {
             }
         }
         intermediate(self)?;
-        self.expect(Token::RightBracket)?;
-        Ok(ret)
+        let span = self.expect_spanned(Token::RightBracket)?;
+        Ok((span, ret))
     }
 
     fn table_key(&mut self) -> Result<Cow<'a, str>, Error> {
@@ -1044,16 +1139,24 @@ impl<'a> Deserializer<'a> {
         self.tokens.eat(expected).map_err(|e| self.token_error(e))
     }
 
+    fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> {
+        self.tokens.eat_spanned(expected).map_err(|e| self.token_error(e))
+    }
+
     fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
         self.tokens.expect(expected).map_err(|e| self.token_error(e))
     }
 
-    fn next(&mut self) -> Result<Option<Token<'a>>, Error> {
-        self.tokens.next().map(|t| t.map(|t| t.1)).map_err(|e| self.token_error(e))
+    fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> {
+        self.tokens.expect_spanned(expected).map_err(|e| self.token_error(e))
     }
 
-    fn peek(&mut self) -> Result<Option<Token<'a>>, Error> {
-        self.tokens.peek().map(|t| t.map(|t| t.1)).map_err(|e| self.token_error(e))
+    fn next(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
+        self.tokens.next().map_err(|e| self.token_error(e))
+    }
+
+    fn peek(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
+        self.tokens.peek().map_err(|e| self.token_error(e))
     }
 
     fn eof(&self) -> Error {
@@ -1300,7 +1403,14 @@ impl<'a> Header<'a> {
 }
 
 #[derive(Debug)]
-enum Value<'a> {
+struct Value<'a> {
+    e: E<'a>,
+    start: usize,
+    end: usize,
+}
+
+#[derive(Debug)]
+enum E<'a> {
     Integer(i64),
     Float(f64),
     Boolean(bool),
@@ -1312,14 +1422,14 @@ enum Value<'a> {
 
 impl<'a> Value<'a> {
     fn same_type(&self, other: &Value<'a>) -> bool {
-        match (self, other) {
-            (&Value::String(..), &Value::String(..)) |
-            (&Value::Integer(..), &Value::Integer(..)) |
-            (&Value::Float(..), &Value::Float(..)) |
-            (&Value::Boolean(..), &Value::Boolean(..)) |
-            (&Value::Datetime(..), &Value::Datetime(..)) |
-            (&Value::Array(..), &Value::Array(..)) |
-            (&Value::InlineTable(..), &Value::InlineTable(..)) => true,
+        match (&self.e, &other.e) {
+            (&E::String(..), &E::String(..)) |
+            (&E::Integer(..), &E::Integer(..)) |
+            (&E::Float(..), &E::Float(..)) |
+            (&E::Boolean(..), &E::Boolean(..)) |
+            (&E::Datetime(..), &E::Datetime(..)) |
+            (&E::Array(..), &E::Array(..)) |
+            (&E::InlineTable(..), &E::InlineTable(..)) => true,
 
             _ => false,
         }
diff --git a/src/lib.rs b/src/lib.rs
index 1c4842b..2f038e8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -169,3 +169,7 @@ mod tokens;
 
 #[doc(hidden)]
 pub mod macros;
+
+mod spanned;
+#[doc(no_inline)]
+pub use spanned::Spanned;
diff --git a/src/ser.rs b/src/ser.rs
index 58c0b42..b8cec35 100644
--- a/src/ser.rs
+++ b/src/ser.rs
@@ -33,7 +33,7 @@ use std::marker;
 use std::rc::Rc;
 
 use serde::ser;
-use datetime::{SERDE_STRUCT_FIELD_NAME, SERDE_STRUCT_NAME};
+use datetime;
 
 /// Serialize the given data structure as a TOML byte vector.
 ///
@@ -924,7 +924,7 @@ impl<'a, 'b> ser::Serializer for &'b mut Serializer<'a> {
 
     fn serialize_struct(self, name: &'static str, _len: usize)
                         -> Result<Self::SerializeStruct, Self::Error> {
-        if name == SERDE_STRUCT_NAME {
+        if name == datetime::NAME {
             self.array_type("datetime")?;
             Ok(SerializeTable::Datetime(self))
         } else {
@@ -1071,7 +1071,7 @@ impl<'a, 'b> ser::SerializeStruct for SerializeTable<'a, 'b> {
     {
         match *self {
             SerializeTable::Datetime(ref mut ser) => {
-                if key == SERDE_STRUCT_FIELD_NAME {
+                if key == datetime::FIELD {
                     value.serialize(DateStrEmitter(&mut *ser))?;
                 } else {
                     return Err(Error::DateInvalid)
diff --git a/src/spanned.rs b/src/spanned.rs
new file mode 100644
index 0000000..abbbd49
--- /dev/null
+++ b/src/spanned.rs
@@ -0,0 +1,140 @@
+//! ```
+//! #[macro_use]
+//! extern crate serde_derive;
+//!
+//! extern crate toml;
+//! use toml::Spanned;
+//!
+//! #[derive(Deserialize)]
+//! struct Value {
+//!     s: Spanned<String>,
+//! }
+//!
+//! fn main() {
+//!     let t = "s = \"value\"\n";
+//!
+//!     let u: Value = toml::from_str(t).unwrap();
+//!
+//!     assert_eq!(u.s.start(), 4);
+//!     assert_eq!(u.s.end(), 11);
+//!     assert_eq!(u.s.get_ref(), "value");
+//!     assert_eq!(u.s.into_inner(), String::from("value"));
+//! }
+//! ```
+
+use serde::{de, ser};
+use std::fmt;
+
+#[doc(hidden)]
+pub const NAME: &'static str = "$__toml_private_Spanned";
+#[doc(hidden)]
+pub const START: &'static str = "$__toml_private_start";
+#[doc(hidden)]
+pub const END: &'static str = "$__toml_private_end";
+#[doc(hidden)]
+pub const VALUE: &'static str = "$__toml_private_value";
+
+/// A spanned value, indicating the range at which it is defined in the source.
+#[derive(Debug)]
+pub struct Spanned<T> {
+    /// The start range.
+    start: usize,
+    /// The end range (exclusive).
+    end: usize,
+    /// The spanned value.
+    value: T,
+}
+
+impl<T> Spanned<T> {
+    /// Access the start of the span of the contained value.
+    pub fn start(&self) -> usize {
+        self.start
+    }
+
+    /// Access the end of the span of the contained value.
+    pub fn end(&self) -> usize {
+        self.end
+    }
+
+    /// Get the span of the contained value.
+    pub fn span(&self) -> (usize, usize) {
+        (self.start, self.end)
+    }
+
+    /// Consumes the spanned value and returns the contained value.
+    pub fn into_inner(self) -> T {
+        self.value
+    }
+
+    /// Returns a reference to the contained value.
+    pub fn get_ref(&self) -> &T {
+        &self.value
+    }
+
+    /// Returns a mutable reference to the contained value.
+    pub fn get_mut(&self) -> &T {
+        &self.value
+    }
+}
+
+impl<'de, T> de::Deserialize<'de> for Spanned<T>
+    where T: de::Deserialize<'de>
+{
+    fn deserialize<D>(deserializer: D) -> Result<Spanned<T>, D::Error>
+        where D: de::Deserializer<'de>
+    {
+        struct SpannedVisitor<T>(::std::marker::PhantomData<T>);
+
+        impl<'de, T> de::Visitor<'de> for SpannedVisitor<T>
+            where T: de::Deserialize<'de>
+        {
+            type Value = Spanned<T>;
+
+            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                formatter.write_str("a TOML spanned")
+            }
+
+            fn visit_map<V>(self, mut visitor: V) -> Result<Spanned<T>, V::Error>
+                where V: de::MapAccess<'de>
+            {
+                if visitor.next_key()? != Some(START) {
+                    return Err(de::Error::custom("spanned start key not found"))
+                }
+
+                let start: usize = visitor.next_value()?;
+
+                if visitor.next_key()? != Some(END) {
+                    return Err(de::Error::custom("spanned end key not found"))
+                }
+
+                let end: usize = visitor.next_value()?;
+
+                if visitor.next_key()? != Some(VALUE) {
+                    return Err(de::Error::custom("spanned value key not found"))
+                }
+
+                let value: T = visitor.next_value()?;
+
+                Ok(Spanned {
+                    start: start,
+                    end: end,
+                    value: value
+                })
+            }
+        }
+
+        let visitor = SpannedVisitor(::std::marker::PhantomData);
+
+        static FIELDS: [&'static str; 3] = [START, END, VALUE];
+        deserializer.deserialize_struct(NAME, &FIELDS, visitor)
+    }
+}
+
+impl<T: ser::Serialize> ser::Serialize for Spanned<T> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: ser::Serializer,
+    {
+        self.value.serialize(serializer)
+    }
+}
diff --git a/src/tokens.rs b/src/tokens.rs
index 3f47f02..1b5a755 100644
--- a/src/tokens.rs
+++ b/src/tokens.rs
@@ -119,21 +119,34 @@ impl<'a> Tokenizer<'a> {
     }
 
     pub fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error> {
-        match self.peek()? {
-            Some((_, ref found)) if expected == *found => {}
-            Some(_) => return Ok(false),
-            None => return Ok(false),
-        }
+        self.eat_spanned(expected).map(|s| s.is_some())
+    }
+
+    /// Eat a value, returning it's span if it was consumed.
+    pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> {
+        let span = match self.peek()? {
+            Some((span, ref found)) if expected == *found => span,
+            Some(_) => return Ok(None),
+            None => return Ok(None),
+        };
+
         drop(self.next());
-        Ok(true)
+        Ok(Some(span))
     }
 
     pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
+        // ignore span
+        let _ = self.expect_spanned(expected)?;
+        Ok(())
+    }
+
+    /// Expect the given token returning its span.
+    pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> {
         let current = self.current();
         match self.next()? {
-            Some((_, found)) => {
+            Some((span, found)) => {
                 if expected == found {
-                    Ok(())
+                    Ok(span)
                 } else {
                     Err(Error::Wanted {
                         at: current,
diff --git a/src/value.rs b/src/value.rs
index 54fa45c..21a8205 100644
--- a/src/value.rs
+++ b/src/value.rs
@@ -12,7 +12,7 @@ use serde::de;
 use serde::de::IntoDeserializer;
 
 pub use datetime::{Datetime, DatetimeParseError};
-use datetime::{DatetimeFromString, SERDE_STRUCT_FIELD_NAME};
+use datetime::{self, DatetimeFromString};
 
 /// Representation of a TOML value.
 #[derive(PartialEq, Clone, Debug)]
@@ -925,7 +925,7 @@ impl<'a, 'de> de::Visitor<'de> for DatetimeOrTable<'a> {
     fn visit_str<E>(self, s: &str) -> Result<bool, E>
         where E: de::Error,
     {
-        if s == SERDE_STRUCT_FIELD_NAME {
+        if s == datetime::FIELD {
             Ok(true)
         } else {
             self.key.push_str(s);
@@ -936,7 +936,7 @@ impl<'a, 'de> de::Visitor<'de> for DatetimeOrTable<'a> {
     fn visit_string<E>(self, s: String) -> Result<bool, E>
         where E: de::Error,
     {
-        if s == SERDE_STRUCT_FIELD_NAME {
+        if s == datetime::FIELD {
             Ok(true)
         } else {
             *self.key = s;