diff options
Diffstat (limited to 'src/content/text.rs')
-rw-r--r-- | src/content/text.rs | 21 |
1 files changed, 14 insertions, 7 deletions
diff --git a/src/content/text.rs b/src/content/text.rs index 73c2d55..433d030 100644 --- a/src/content/text.rs +++ b/src/content/text.rs @@ -52,12 +52,19 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// |qwe /// ``` fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - if let Code::None = code { - (State::Ok, None) - } else { - tokenizer.enter(TokenType::Data); - tokenizer.consume(code); - (State::Fn(Box::new(in_data)), None) + match code { + Code::None => (State::Ok, None), + Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => { + tokenizer.enter(TokenType::LineEnding); + tokenizer.consume(code); + tokenizer.exit(TokenType::LineEnding); + (State::Fn(Box::new(start)), None) + } + _ => { + tokenizer.enter(TokenType::Data); + tokenizer.consume(code); + (State::Fn(Box::new(in_data)), None) + } } } @@ -73,7 +80,7 @@ fn in_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { (State::Ok, None) } // To do: somehow get these markers from constructs. - Code::Char('&' | '\\' | '<') => { + Code::CarriageReturnLineFeed | Code::Char('\r' | '\n' | '&' | '\\' | '<') => { tokenizer.exit(TokenType::Data); start(tokenizer, code) } |