aboutsummaryrefslogtreecommitdiffstats
path: root/src/content
diff options
context:
space:
mode:
Diffstat (limited to 'src/content')
-rw-r--r--src/content/text.rs21
1 files changed, 14 insertions, 7 deletions
diff --git a/src/content/text.rs b/src/content/text.rs
index 73c2d55..433d030 100644
--- a/src/content/text.rs
+++ b/src/content/text.rs
@@ -52,12 +52,19 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// |qwe
/// ```
fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- if let Code::None = code {
- (State::Ok, None)
- } else {
- tokenizer.enter(TokenType::Data);
- tokenizer.consume(code);
- (State::Fn(Box::new(in_data)), None)
+ match code {
+ Code::None => (State::Ok, None),
+ Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+ tokenizer.enter(TokenType::LineEnding);
+ tokenizer.consume(code);
+ tokenizer.exit(TokenType::LineEnding);
+ (State::Fn(Box::new(start)), None)
+ }
+ _ => {
+ tokenizer.enter(TokenType::Data);
+ tokenizer.consume(code);
+ (State::Fn(Box::new(in_data)), None)
+ }
}
}
@@ -73,7 +80,7 @@ fn in_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
(State::Ok, None)
}
// To do: somehow get these markers from constructs.
- Code::Char('&' | '\\' | '<') => {
+ Code::CarriageReturnLineFeed | Code::Char('\r' | '\n' | '&' | '\\' | '<') => {
tokenizer.exit(TokenType::Data);
start(tokenizer, code)
}