1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
// To do: pass token types in.
use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('<') => {
tokenizer.enter(TokenType::DefinitionDestination);
tokenizer.enter(TokenType::DefinitionDestinationLiteral);
tokenizer.enter(TokenType::DefinitionDestinationLiteralMarker);
tokenizer.consume(code);
tokenizer.exit(TokenType::DefinitionDestinationLiteralMarker);
(State::Fn(Box::new(enclosed_before)), None)
}
Code::None | Code::CarriageReturnLineFeed | Code::VirtualSpace | Code::Char(')') => {
(State::Nok, None)
}
Code::Char(char) if char.is_ascii_control() => (State::Nok, None),
Code::Char(_) => {
tokenizer.enter(TokenType::DefinitionDestination);
tokenizer.enter(TokenType::DefinitionDestinationRaw);
tokenizer.enter(TokenType::DefinitionDestinationString);
// To do: link.
tokenizer.enter(TokenType::ChunkString);
raw(tokenizer, code, 0)
}
}
}
/// To do.
fn enclosed_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if let Code::Char('>') = code {
tokenizer.enter(TokenType::DefinitionDestinationLiteralMarker);
tokenizer.consume(code);
tokenizer.exit(TokenType::DefinitionDestinationLiteralMarker);
tokenizer.exit(TokenType::DefinitionDestinationLiteral);
tokenizer.exit(TokenType::DefinitionDestination);
(State::Ok, None)
} else {
tokenizer.enter(TokenType::DefinitionDestinationString);
// To do: link.
tokenizer.enter(TokenType::ChunkString);
enclosed(tokenizer, code)
}
}
/// To do.
fn enclosed(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
tokenizer.exit(TokenType::ChunkString);
tokenizer.exit(TokenType::DefinitionDestinationString);
enclosed_before(tokenizer, code)
}
Code::None | Code::CarriageReturnLineFeed | Code::Char('\r' | '\n' | '<') => {
(State::Nok, None)
}
Code::Char('\\') => {
tokenizer.consume(code);
(State::Fn(Box::new(enclosed_escape)), None)
}
_ => {
tokenizer.consume(code);
(State::Fn(Box::new(enclosed)), None)
}
}
}
/// To do.
fn enclosed_escape(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('<' | '>' | '\\') => {
tokenizer.consume(code);
(State::Fn(Box::new(enclosed)), None)
}
_ => enclosed(tokenizer, code),
}
}
/// To do.
// To do: these arms can be improved?
fn raw(tokenizer: &mut Tokenizer, code: Code, balance: usize) -> StateFnResult {
// To do: configurable.
let limit = usize::MAX;
match code {
Code::Char('(') if balance >= limit => (State::Nok, None),
Code::Char('(') => {
tokenizer.consume(code);
(
State::Fn(Box::new(move |t, c| raw(t, c, balance + 1))),
None,
)
}
Code::Char(')') if balance == 0 => {
tokenizer.exit(TokenType::ChunkString);
tokenizer.exit(TokenType::DefinitionDestinationString);
tokenizer.exit(TokenType::DefinitionDestinationRaw);
tokenizer.exit(TokenType::DefinitionDestination);
(State::Ok, Some(vec![code]))
}
Code::Char(')') => {
tokenizer.consume(code);
(
State::Fn(Box::new(move |t, c| raw(t, c, balance - 1))),
None,
)
}
Code::None
| Code::CarriageReturnLineFeed
| Code::VirtualSpace
| Code::Char('\t' | '\r' | '\n' | ' ')
if balance > 0 =>
{
(State::Nok, None)
}
Code::None
| Code::CarriageReturnLineFeed
| Code::VirtualSpace
| Code::Char('\t' | '\r' | '\n' | ' ') => {
tokenizer.exit(TokenType::ChunkString);
tokenizer.exit(TokenType::DefinitionDestinationString);
tokenizer.exit(TokenType::DefinitionDestinationRaw);
tokenizer.exit(TokenType::DefinitionDestination);
(State::Ok, Some(vec![code]))
}
Code::Char(char) if char.is_ascii_control() => (State::Nok, None),
Code::Char('\\') => {
tokenizer.consume(code);
(
State::Fn(Box::new(move |t, c| raw_escape(t, c, balance))),
None,
)
}
Code::Char(_) => {
tokenizer.consume(code);
(State::Fn(Box::new(move |t, c| raw(t, c, balance))), None)
}
}
}
/// To do.
fn raw_escape(tokenizer: &mut Tokenizer, code: Code, balance: usize) -> StateFnResult {
match code {
Code::Char('(' | ')' | '\\') => {
tokenizer.consume(code);
(
State::Fn(Box::new(move |t, c| raw(t, c, balance + 1))),
None,
)
}
_ => raw(tokenizer, code, balance),
}
}
|
