Refactor to improve tokenizer, add docs

3 files changed, 24 insertions, 19 deletions

diff --git a/src/content/flow.rs b/src/content/flow.rs
index 481c8ff..6283fef 100644
--- a/src/content/flow.rs
+++ b/src/content/flow.rs
@@ -94,14 +94,16 @@ fn blank_line_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 fn initial_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
     match code {
         Code::None => (State::Ok, None),
-        _ => tokenizer.attempt_7(
-            code_indented,
-            code_fenced,
-            html_flow,
-            heading_atx,
-            thematic_break,
-            definition,
-            heading_setext,
+        _ => tokenizer.attempt_n(
+            vec![
+                Box::new(code_indented),
+                Box::new(code_fenced),
+                Box::new(html_flow),
+                Box::new(heading_atx),
+                Box::new(thematic_break),
+                Box::new(definition),
+                Box::new(heading_setext),
+            ],
             |ok| Box::new(if ok { after } else { before_paragraph }),
         )(tokenizer, code),
     }
diff --git a/src/content/string.rs b/src/content/string.rs
index 3338c90..53e88b1 100644
--- a/src/content/string.rs
+++ b/src/content/string.rs
@@ -33,9 +33,10 @@ const MARKERS: [Code; 2] = [
 pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
     match code {
         Code::None => (State::Ok, None),
-        _ => tokenizer.attempt_2(character_reference, character_escape, |ok| {
-            Box::new(if ok { start } else { before_data })
-        })(tokenizer, code),
+        _ => tokenizer.attempt_n(
+            vec![Box::new(character_reference), Box::new(character_escape)],
+            |ok| Box::new(if ok { start } else { before_data }),
+        )(tokenizer, code),
     }
 }
 
diff --git a/src/content/text.rs b/src/content/text.rs
index 857e9a0..1224064 100644
--- a/src/content/text.rs
+++ b/src/content/text.rs
@@ -45,14 +45,16 @@ const MARKERS: [Code; 5] = [
 pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
     match code {
         Code::None => (State::Ok, None),
-        _ => tokenizer.attempt_7(
-            character_reference,
-            character_escape,
-            hard_break_escape,
-            hard_break_trailing,
-            autolink,
-            html_text,
-            code_text,
+        _ => tokenizer.attempt_n(
+            vec![
+                Box::new(character_reference),
+                Box::new(character_escape),
+                Box::new(hard_break_escape),
+                Box::new(hard_break_trailing),
+                Box::new(autolink),
+                Box::new(html_text),
+                Box::new(code_text),
+            ],
             |ok| Box::new(if ok { start } else { before_data }),
         )(tokenizer, code),
     }