//! Incremental UTF-8 decoder with Node `StringDecoder` semantics: incomplete trailing //! sequences are kept for the next chunk, invalid bytes become U+FFFD. #[derive(Default)] pub struct Utf8Decoder { pending: Vec, } fn sequence_length(first: u8) -> usize { match first { 0x00..=0x7f => 1, 0xc2..=0xdf => 2, 0xe0..=0xef => 3, 0xf0..=0xf4 => 4, _ => 0, } } impl Utf8Decoder { pub fn write(&mut self, chunk: &[u8]) -> String { let mut bytes = std::mem::take(&mut self.pending); bytes.extend_from_slice(chunk); // Keep an incomplete (but so far valid) sequence at the end for the next chunk. let mut cut = bytes.len(); for back in 1..=3.min(bytes.len()) { let index = bytes.len() - back; let byte = bytes[index]; if byte & 0xc0 == 0x80 { continue; } let needed = sequence_length(byte); if needed > back { cut = index; } break; } self.pending = bytes.split_off(cut); String::from_utf8_lossy(&bytes).into_owned() } pub fn end(&mut self) -> String { let rest = std::mem::take(&mut self.pending); if rest.is_empty() { String::new() } else { String::from_utf8_lossy(&rest).into_owned() } } } #[cfg(test)] mod tests { use super::*; #[test] fn split_multibyte() { let text = "Příliš žluťoučký kůň 🐎"; let bytes = text.as_bytes(); for split in 0..bytes.len() { let mut decoder = Utf8Decoder::default(); let mut out = decoder.write(&bytes[..split]); out.push_str(&decoder.write(&bytes[split..])); out.push_str(&decoder.end()); assert_eq!(out, text, "split at {split}"); } let mut decoder = Utf8Decoder::default(); assert_eq!(decoder.write(&[0xff, b'a']), "\u{fffd}a"); assert_eq!(decoder.write(&[0xe2, 0x82]), ""); assert_eq!(decoder.end(), "\u{fffd}"); } }