1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
//! Several helpers to parse whitespace (`space_or_tab`).
//!
//! ## References
//!
//! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js)
use crate::tokenizer::{Code, State, StateFn, StateFnResult, TokenType, Tokenizer};
/// Optional `space_or_tab`
///
/// ```bnf
/// space_or_tab_opt ::= *( ' ' '\t' )
/// ```
pub fn space_or_tab_opt() -> Box<StateFn> {
space_or_tab_min_max(0, usize::MAX)
}
/// Between `x` and `y` `space_or_tab`
///
/// ```bnf
/// space_or_tab_min_max ::= x*y( ' ' '\t' )
/// ```
pub fn space_or_tab_min_max(min: usize, max: usize) -> Box<StateFn> {
space_or_tab(TokenType::Whitespace, min, max)
}
/// Between `x` and `y` `space_or_tab`, with the given token type.
///
/// ```bnf
/// space_or_tab ::= x*y( ' ' '\t' )
/// ```
pub fn space_or_tab(kind: TokenType, min: usize, max: usize) -> Box<StateFn> {
Box::new(move |t, c| start(t, c, kind, min, max))
}
/// Before whitespace.
///
/// ```markdown
/// alpha| bravo
/// ```
fn start(
tokenizer: &mut Tokenizer,
code: Code,
kind: TokenType,
min: usize,
max: usize,
) -> StateFnResult {
match code {
Code::VirtualSpace | Code::Char('\t' | ' ') if max > 0 => {
tokenizer.enter(kind.clone());
tokenizer.consume(code);
(
State::Fn(Box::new(move |tokenizer, code| {
inside(tokenizer, code, kind, min, max, 1)
})),
None,
)
}
_ => (
if min == 0 { State::Ok } else { State::Nok },
Some(vec![code]),
),
}
}
/// In whitespace.
///
/// ```markdown
/// alpha |bravo
/// alpha | bravo
/// ```
fn inside(
tokenizer: &mut Tokenizer,
code: Code,
kind: TokenType,
min: usize,
max: usize,
size: usize,
) -> StateFnResult {
match code {
Code::VirtualSpace | Code::Char('\t' | ' ') if size < max => {
tokenizer.consume(code);
(
State::Fn(Box::new(move |tokenizer, code| {
inside(tokenizer, code, kind, min, max, size + 1)
})),
None,
)
}
_ => {
tokenizer.exit(kind);
(
if size >= min { State::Ok } else { State::Nok },
Some(vec![code]),
)
}
}
}
|
