From 182467c1d393dee2081ff80f1c049cb145f23123 Mon Sep 17 00:00:00 2001
From: Titus Wormer <tituswormer@gmail.com>
Date: Mon, 20 Jun 2022 18:53:38 +0200
Subject: Add support for BOM

---
 src/subtokenize.rs | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'src/subtokenize.rs')

diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index 4a29a01..0623a37 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -15,6 +15,10 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) {
     let mut link_to_info: HashMap<usize, (usize, usize, usize)> = HashMap::new();
     let mut done = true;
 
+    if events.is_empty() {
+        return (events, true);
+    }
+
     while index < events.len() {
         let event = &events[index];
 
-- 
cgit