From 94c95bd15e63520fc2844ffb1bc387d1bc9c79ae Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 27 Oct 2018 12:41:52 -0700 Subject: Allow whitespace after line ending backslash (#162) --- src/tokens.rs | 19 ++++++++++++++++++- .../tests/invalid/string-bad-line-ending-escape.toml | 3 +++ test-suite/tests/valid/multiline-string.json | 8 ++++++++ test-suite/tests/valid/multiline-string.toml | 11 +++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 test-suite/tests/invalid/string-bad-line-ending-escape.toml diff --git a/src/tokens.rs b/src/tokens.rs index 15c3b41..382c1ec 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -364,7 +364,24 @@ impl<'a> Tokenizer<'a> { let len = if c == 'u' {4} else {8}; val.push(me.hex(start, i, len)?); } - Some((_, '\n')) if multi => { + Some((i, c @ ' ')) | + Some((i, c @ '\t')) | + Some((i, c @ '\n')) if multi => { + if c != '\n' { + while let Some((_, ch)) = me.chars.clone().next() { + match ch { + ' ' | '\t' => { + me.chars.next(); + continue + }, + '\n' => { + me.chars.next(); + break + }, + _ => return Err(Error::InvalidEscape(i, c)), + } + } + } while let Some((_, ch)) = me.chars.clone().next() { match ch { ' ' | '\t' | '\n' => { diff --git a/test-suite/tests/invalid/string-bad-line-ending-escape.toml b/test-suite/tests/invalid/string-bad-line-ending-escape.toml new file mode 100644 index 0000000..32e2c48 --- /dev/null +++ b/test-suite/tests/invalid/string-bad-line-ending-escape.toml @@ -0,0 +1,3 @@ +invalid-escape = """\ + This string has a non whitespace-character after the line ending escape. \ a +""" diff --git a/test-suite/tests/valid/multiline-string.json b/test-suite/tests/valid/multiline-string.json index 075bf50..3223bae 100644 --- a/test-suite/tests/valid/multiline-string.json +++ b/test-suite/tests/valid/multiline-string.json @@ -15,6 +15,10 @@ "type": "string", "value": "" }, + "multiline_empty_five": { + "type": "string", + "value": "" + }, "equivalent_one": { "type": "string", "value": "The quick brown fox jumps over the lazy dog." @@ -26,5 +30,9 @@ "equivalent_three": { "type": "string", "value": "The quick brown fox jumps over the lazy dog." + }, + "equivalent_four": { + "type": "string", + "value": "The quick brown fox jumps over the lazy dog." } } diff --git a/test-suite/tests/valid/multiline-string.toml b/test-suite/tests/valid/multiline-string.toml index 15b1143..2c4237f 100644 --- a/test-suite/tests/valid/multiline-string.toml +++ b/test-suite/tests/valid/multiline-string.toml @@ -7,6 +7,11 @@ multiline_empty_four = """\ \ \ """ +multiline_empty_five = """\ + \ + \ + \ + """ equivalent_one = "The quick brown fox jumps over the lazy dog." equivalent_two = """ @@ -21,3 +26,9 @@ equivalent_three = """\ fox jumps over \ the lazy dog.\ """ + +equivalent_four = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ -- cgit v1.2.3