diff options
| -rw-r--r-- | src/content/document.rs | 527 | ||||
| -rw-r--r-- | src/tokenizer.rs | 11 |
2 files changed, 278 insertions, 260 deletions
diff --git a/src/content/document.rs b/src/content/document.rs index 2006021..82ccd65 100644 --- a/src/content/document.rs +++ b/src/content/document.rs @@ -6,7 +6,7 @@ //! The constructs found in flow are: //! //! * [Block quote][crate::construct::block_quote] -//! * List +//! * [List][crate::construct::list] use crate::construct::{ block_quote::{cont as block_quote_cont, end as block_quote_end, start as block_quote}, @@ -17,7 +17,8 @@ use crate::parser::ParseState; use crate::subtokenize::subtokenize; use crate::token::Token; use crate::tokenizer::{ - Code, ContainerState, Event, EventType, Point, State, StateFn, StateFnResult, Tokenizer, + Code, Container, ContainerState, Event, EventType, Point, State, StateFn, StateFnResult, + Tokenizer, }; use crate::util::edit_map::EditMap; use crate::util::{ @@ -27,21 +28,52 @@ use crate::util::{ }; use std::collections::HashSet; +/// Phases where we can exit containers. #[derive(Debug, PartialEq)] -enum Container { - BlockQuote, - ListItem, +enum Phase { + /// After parsing a line of lazy flow which resulted in something that + /// exits containers before the line. + /// + /// ```markdown + /// | * a + /// > | ```js + /// ^ + /// | b + /// | ``` + /// ``` + After, + /// When a new container replaces an existing container. + /// + /// ```markdown + /// | * a + /// > | > b + /// ^ + /// ``` + Prefix, + /// After everything. + /// + /// ```markdown + /// > | * a + /// ^ + /// ``` + Eof, } +/// State needed to parse document. struct DocumentInfo { + /// Number of containers that have continued. continued: usize, + /// Index into `tokenizer.events` we need to track. index: usize, - paragraph_before: bool, - interrupt_before: bool, + /// Events of containers added back later. inject: Vec<(Vec<Event>, Vec<Event>)>, - stack: Vec<Container>, - states: Vec<ContainerState>, - stack_close: Vec<Container>, + /// The value of the previous line of flow’s `interrupt`. + interrupt_before: bool, + /// Whether the previous line of flow was a paragraph. + paragraph_before: bool, + /// Current containers. + stack: Vec<ContainerState>, + /// Current flow state function. next: Box<StateFn>, } @@ -82,6 +114,13 @@ pub fn document(parse_state: &mut ParseState, point: Point, index: usize) -> Vec result.0 } +/// Before document. +// +/// ```markdown +/// > | * a +/// ^ +/// | > b +/// ``` fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { let info = DocumentInfo { index: 0, @@ -91,15 +130,19 @@ fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { paragraph_before: false, interrupt_before: false, stack: vec![], - states: vec![], - stack_close: vec![], }; line_start(tokenizer, code, info) } -/// Start of a new line. +/// Start of a line. +// +/// ```markdown +/// > | * a +/// ^ +/// > | > b +/// ^ +/// ``` fn line_start(tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo) -> StateFnResult { - println!("line_start"); info.index = tokenizer.events.len(); info.inject.push((vec![], vec![])); info.continued = 0; @@ -109,34 +152,26 @@ fn line_start(tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo) -> } /// Before existing containers. +// +/// ```markdown +/// | * a +/// > | > b +/// ^ +/// ``` fn container_existing_before( tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo, ) -> StateFnResult { - println!("container_existing_before"); - - // First we iterate through the open blocks, starting with the root - // document, and descending through last children down to the last open - // block. - // Each block imposes a condition that the line must satisfy if the block - // is to remain open. - // For example, a block quote requires a `>` character. - // A paragraph requires a non-blank line. - // In this phase we may match all or just some of the open blocks. - // But we cannot close unmatched blocks yet, because we may have a lazy - // continuation line. + // If there are more existing containers, check whether the next one continues. if info.continued < info.stack.len() { - let kind = &info.stack[info.continued]; - let container = info.states.remove(info.continued); - tokenizer.container = Some(container); - let cont = match kind { + let container = info.stack.remove(info.continued); + let cont = match container.kind { Container::BlockQuote => block_quote_cont, Container::ListItem => list_item_const, }; - // tokenizer.container = Some(&mut info.states[info.continued]); - // To do: state? + tokenizer.container = Some(container); tokenizer.attempt(cont, move |ok| { if ok { Box::new(|t, c| container_existing_after(t, c, info)) @@ -144,97 +179,124 @@ fn container_existing_before( Box::new(|t, c| container_existing_missing(t, c, info)) } })(tokenizer, code) - } else { - // Done. + } + // Otherwise, check new containers. + else { container_new_before(tokenizer, code, info) } } +/// At a missing, existing containers. +// +/// ```markdown +/// | * a +/// > | > b +/// ^ +/// ``` fn container_existing_missing( tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo, ) -> StateFnResult { let container = tokenizer.container.take().unwrap(); - info.states.insert(info.continued, container); + info.stack.insert(info.continued, container); container_new_before(tokenizer, code, info) } +/// After an existing container. +// +/// ```markdown +/// | * a +/// > | b +/// ^ +/// ``` fn container_existing_after( tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo, ) -> StateFnResult { - println!("container_existing_after"); let container = tokenizer.container.take().unwrap(); - info.states.insert(info.continued, container); + info.stack.insert(info.continued, container); info.continued += 1; container_existing_before(tokenizer, code, info) } +/// Before a new container. +// +/// ```markdown +/// > | * a +/// ^ +/// > | > b +/// ^ +/// ``` fn container_new_before( tokenizer: &mut Tokenizer, code: Code, info: DocumentInfo, ) -> StateFnResult { - println!("container_new_before"); - // Next, after consuming the continuation markers for existing blocks, we - // look for new block starts (e.g. `>` for a block quote). - // If we encounter a new block start, we close any blocks unmatched in - // step 1 before creating the new block as a child of the last matched - // block. + // If we have completely continued, restore the flow’s past `interrupt` + // status. if info.continued == info.stack.len() { - // If we have concrete content, such as block HTML or fenced code, - // we can’t have containers “pierce” into them, so we can immediately - // start. + tokenizer.interrupt = info.interrupt_before; + + // …and if we’re in a concrete construct, new containers can’t “pierce” + // into them. if tokenizer.concrete { - println!(" concrete"); return containers_after(tokenizer, code, info); } - - println!( - " set interrupt to {:?} because we have continued (was: {:?})", - info.interrupt_before, tokenizer.interrupt - ); - tokenizer.interrupt = info.interrupt_before; - - // // If we do have flow, it could still be a blank line, - // // but we’d be interrupting it w/ a new container if there’s a current - // // construct. - // self.interrupt = Boolean( - // childFlow.currentConstruct && !childFlow._gfmTableDynamicInterruptHack - // ) } - tokenizer.container = Some(ContainerState::default()); - // Check if there is a new container. + // Check for a new container. + // Block quote? + tokenizer.container = Some(ContainerState { + kind: Container::BlockQuote, + blank_initial: false, + size: 0, + }); + tokenizer.attempt(block_quote, move |ok| { if ok { - Box::new(|t, c| container_new_after(t, c, info, Container::BlockQuote)) + Box::new(|t, c| container_new_after(t, c, info)) } else { Box::new(|tokenizer, code| { - tokenizer.container = Some(ContainerState::default()); - tokenizer.attempt(list_item, move |ok| { - if ok { - Box::new(|t, c| container_new_after(t, c, info, Container::ListItem)) + // List item? + tokenizer.container = Some(ContainerState { + kind: Container::ListItem, + blank_initial: false, + size: 0, + }); + + tokenizer.attempt(list_item, |ok| { + let func = if ok { + container_new_after } else { - Box::new(|t, c| containers_after(t, c, info)) - } + containers_after + }; + Box::new(move |t, c| func(t, c, info)) })(tokenizer, code) }) } })(tokenizer, code) } +/// After a new container. +// +/// ```markdown +/// > | * a +/// ^ +/// > | > b +/// ^ +/// ``` fn container_new_after( tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo, - kind: Container, ) -> StateFnResult { + let container = tokenizer.container.take().unwrap(); + // Remove from the event stack. // We’ll properly add exits at different points manually. - let end = match kind { + let end = match container.kind { Container::BlockQuote => block_quote_end, Container::ListItem => list_item_end, }; @@ -261,212 +323,114 @@ fn container_new_after( index += 1; } - if info.continued < info.stack.len() { - info.stack_close - .append(&mut info.stack.drain(info.continued..).collect::<Vec<_>>()); - info.states.truncate(info.continued); - info = line_end(tokenizer, info, false, true); + // If we did not continue all existing containers, and there is a new one, + // close the flow and those containers. + if info.continued != info.stack.len() { + info = exit_containers(tokenizer, info, &Phase::Prefix); tokenizer.expect(code, true); } - let container = tokenizer.container.take().unwrap(); - info.states.push(container); - info.stack.push(kind); - info.continued = info.stack.len(); // To do: `+= 1`? - println!( - " set `interrupt`, `info.interrupt_before: false` because we have new containers (before: {:?}, {:?})", - info.interrupt_before, - tokenizer.interrupt - ); + // Try another new container. + info.stack.push(container); + info.continued += 1; info.interrupt_before = false; - tokenizer.interrupt = info.interrupt_before; + tokenizer.interrupt = false; container_new_before(tokenizer, code, info) } +/// After containers, before flow. +// +/// ```markdown +/// > | * a +/// ^ +/// > | > b +/// ^ +/// ``` fn containers_after( tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo, ) -> StateFnResult { - println!("containers_after"); - - // Add all container events we parsed. - let mut containers = tokenizer.events.drain(info.index..).collect::<Vec<_>>(); - info.inject.last_mut().unwrap().0.append(&mut containers); + // Store the container events we parsed. + info.inject + .last_mut() + .unwrap() + .0 + .append(&mut tokenizer.events.drain(info.index..).collect::<Vec<_>>()); tokenizer.lazy = info.continued != info.stack.len(); - println!( - " restoring interrupt: {:?} (was: {:?})", - info.interrupt_before, tokenizer.interrupt - ); tokenizer.interrupt = info.interrupt_before; - - // Define start. tokenizer.define_skip(tokenizer.point.clone(), tokenizer.index); - flow_start(tokenizer, code, info) -} - -fn flow_start(tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo) -> StateFnResult { - println!("flow_start"); - let state = info.next; - info.next = Box::new(flow); // This is weird but Rust needs a function there. - - tokenizer.go_until(state, eol, move |(state, remainder)| { - ( - State::Fn(Box::new(move |t, c| flow_end(t, c, info, state))), - remainder, - ) - })(tokenizer, code) + info.next = Box::new(flow); + + // Parse flow, pausing after eols. + tokenizer.go_until( + state, + |code| matches!(code, Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')), + move |(state, remainder)| { + ( + State::Fn(Box::new(move |t, c| flow_end(t, c, info, state))), + remainder, + ) + }, + )(tokenizer, code) } +/// After flow (after eol or at eof). +// +/// ```markdown +/// | * a +/// > | > b +/// ^ ^ +/// ``` fn flow_end( tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo, result: State, ) -> StateFnResult { - println!("flow_end: lazy? {:?}", tokenizer.lazy); - - // To do: clean this! - let index = tokenizer.events.len(); - let index = if index > 0 { - skip::opt_back(&tokenizer.events, index - 1, &[Token::LineEnding]) - } else { - 0 - }; - - let paragraph = if index > 0 { - let ev = &tokenizer.events[index]; - ev.point.offset + 1 >= tokenizer.point.offset - && ev.token_type == Token::Paragraph - && !(matches!( - tokenizer.previous, - Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') - ) && matches!(code, Code::None)) - } else { - false - }; - - let mut lazy = false; - - if tokenizer.lazy { - println!("this line was lazy."); - - if info.paragraph_before && paragraph { - println!("it was another paragraph, which is allowed."); - lazy = true; - } else { - println!( - "it was something else (prev: {:?}, cur: {:?}), which is not allowed.", - info.paragraph_before, paragraph - ); - } + let paragraph = !tokenizer.events.is_empty() + && tokenizer.events[skip::opt_back( + &tokenizer.events, + tokenizer.events.len() - 1, + &[Token::LineEnding], + )] + .token_type + == Token::Paragraph; + + if tokenizer.lazy + && info.paragraph_before + && paragraph + && !(matches!( + tokenizer.previous, + Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') + ) && matches!(code, Code::None)) + { + info.continued = info.stack.len(); } - if !lazy && info.continued < info.stack.len() { - info.stack_close - .append(&mut info.stack.drain(info.continued..).collect::<Vec<_>>()); - info.states.truncate(info.continued); + if info.continued != info.stack.len() { + info = exit_containers(tokenizer, info, &Phase::After); + tokenizer.expect(code, true); } - info = line_end(tokenizer, info, false, false); - tokenizer.expect(code, true); - info.paragraph_before = paragraph; info.interrupt_before = tokenizer.interrupt; match result { State::Ok => { - info.stack_close - .append(&mut info.stack.drain(..).collect::<Vec<_>>()); - info = line_end(tokenizer, info, true, false); - - let mut map = EditMap::new(); - let mut line_index = 0; - let mut index = 0; - - println!("injections: {:#?}", info.inject); - - let add = info.inject[line_index].0.clone(); - let mut first_line_ending_in_run: Option<usize> = None; - println!("inject:enters:0: {:?}", add.len()); - map.add(0, 0, add); - - while index < tokenizer.events.len() { - let event = &tokenizer.events[index]; - - if event.token_type == Token::LineEnding - || event.token_type == Token::BlankLineEnding - { - if event.event_type == EventType::Enter { - first_line_ending_in_run = first_line_ending_in_run.or(Some(index)); - let mut add = info.inject[line_index].1.clone(); - let mut index = 0; - while index < add.len() { - add[index].point = event.point.clone(); - add[index].index = event.index; - index += 1; - } - if !add.is_empty() { - println!( - "inject:exits:at-{:?}: {:?}", - first_line_ending_in_run, - add.len() - ); - map.add(first_line_ending_in_run.unwrap(), 0, add); - } - } else { - line_index += 1; - let add = info.inject[line_index].0.clone(); - if !add.is_empty() { - // No longer empty. - first_line_ending_in_run = None; - println!("inject:enters:at-{:?}: {:?}", index + 1, add.len()); - map.add(index + 1, 0, add); - } - } - } else if event.token_type == Token::SpaceOrTab { - // Empty to allow whitespace in blank lines. - } else { - first_line_ending_in_run = None; - } - - index += 1; + if !info.stack.is_empty() { + info.continued = 0; + info = exit_containers(tokenizer, info, &Phase::Eof); } - let mut add = info.inject[line_index].1.clone(); - println!("inject:exits:tail-{:?}: {:?}", index, add.len()); - let mut deep_index = 0; - while deep_index < add.len() { - add[deep_index].point = tokenizer.point.clone(); - add[deep_index].index = tokenizer.index; - deep_index += 1; - } - map.add(index, 0, add); - - tokenizer.events = map.consume(&mut tokenizer.events); - let mut index = 0; - - println!("document:after: {:?}", tokenizer.events.len()); - while index < tokenizer.events.len() { - let event = &tokenizer.events[index]; - println!( - "ev: {:?} {:?} {:?} {:?} {:?} {:?}", - index, - event.event_type, - event.token_type, - event.content_type, - event.previous, - event.next - ); - index += 1; - } + tokenizer.events = resolve(tokenizer, &info); (State::Ok, Some(vec![code])) } - State::Nok => unreachable!("handle nok in `flow`?"), + State::Nok => unreachable!("unexpected `nok` from flow"), State::Fn(func) => { info.next = func; line_start(tokenizer, code, info) @@ -474,22 +438,16 @@ fn flow_end( } } -fn line_end( +/// Close containers (and flow if needed). +fn exit_containers( tokenizer: &mut Tokenizer, mut info: DocumentInfo, - eof: bool, - containers_before: bool, + phase: &Phase, ) -> DocumentInfo { - let mut stack_close = info.stack_close.drain(..).collect::<Vec<_>>(); - println!("line_end: {:?}", stack_close); - - if stack_close.is_empty() { - return info; - } + let mut stack_close = info.stack.drain(info.continued..).collect::<Vec<_>>(); // So, we’re at the end of a line, but we need to close the *previous* line. - if !eof { - println!("closing previous flow"); + if *phase != Phase::Eof { tokenizer.define_skip(tokenizer.point.clone(), tokenizer.index); let mut current_events = tokenizer.events.drain(info.index..).collect::<Vec<_>>(); let next = info.next; @@ -498,7 +456,7 @@ fn line_end( assert!(matches!(result.0, State::Ok)); assert!(result.1.is_none()); - if containers_before { + if *phase == Phase::Prefix { info.index = tokenizer.events.len(); } @@ -508,8 +466,8 @@ fn line_end( let mut exits: Vec<Event> = vec![]; while !stack_close.is_empty() { - let kind = stack_close.pop().unwrap(); - let end = match kind { + let container = stack_close.pop().unwrap(); + let end = match container.kind { Container::BlockQuote => block_quote_end, Container::ListItem => list_item_end, }; @@ -535,18 +493,69 @@ fn line_end( } } - let index = info.inject.len() - (if eof { 1 } else { 2 }); + let index = info.inject.len() - (if *phase == Phase::Eof { 1 } else { 2 }); info.inject[index].1.append(&mut exits); - - println!( - " setting `info.interrupt_before: false` (before: {:?})", - info.interrupt_before - ); info.interrupt_before = false; info } -fn eol(code: Code) -> bool { - matches!(code, Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')) +// Inject the container events. +fn resolve(tokenizer: &mut Tokenizer, info: &DocumentInfo) -> Vec<Event> { + let mut map = EditMap::new(); + let mut line_index = 0; + let mut index = 0; + + let add = info.inject[line_index].0.clone(); + let mut first_line_ending_in_run: Option<usize> = None; + map.add(0, 0, add); + + while index < tokenizer.events.len() { + let event = &tokenizer.events[index]; + + if event.token_type == Token::LineEnding || event.token_type == Token::BlankLineEnding { + if event.event_type == EventType::Enter { + first_line_ending_in_run = first_line_ending_in_run.or(Some(index)); + let mut add = info.inject[line_index].1.clone(); + let mut index = 0; + while index < add.len() { + add[index].point = event.point.clone(); + add[index].index = event.index; + index += 1; + } + if !add.is_empty() { + map.add(first_line_ending_in_run.unwrap(), 0, add); + } + } else { + line_index += 1; + let add = info.inject[line_index].0.clone(); + if !add.is_empty() { + // No longer empty. + first_line_ending_in_run = None; + map.add(index + 1, 0, add); + } + } + } else if event.token_type == Token::SpaceOrTab { + // Empty to allow whitespace in blank lines. + } else { + first_line_ending_in_run = None; + } + + index += 1; + } + + let mut add = info.inject[line_index].1.clone(); + let mut index = 0; + while index < add.len() { + add[index].point = tokenizer.point.clone(); + add[index].index = tokenizer.index; + index += 1; + } + map.add( + first_line_ending_in_run.unwrap_or(tokenizer.events.len()), + 0, + add, + ); + + map.consume(&mut tokenizer.events) } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 25efaac..17622e7 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -130,12 +130,21 @@ pub struct Media { pub id: String, } +/// Supported containers. +#[derive(Debug, PartialEq)] +pub enum Container { + BlockQuote, + ListItem, +} + /// Info used to tokenize the current container. /// /// This info is shared between the initial construct and its continuation. /// It’s only used for list items. -#[derive(Default, Debug)] +#[derive(Debug)] pub struct ContainerState { + /// Kind. + pub kind: Container, /// Whether the first (and all future) lines were blank. pub blank_initial: bool, /// The size of the initial construct. |