// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. use crate::tables::grapheme::GraphemeCat; use core::cmp; /// External iterator for grapheme clusters and byte offsets. /// /// This struct is created by the [`grapheme_indices`] method on the [`UnicodeSegmentation`] /// trait. See its documentation for more. /// /// [`grapheme_indices`]: trait.UnicodeSegmentation.html#tymethod.grapheme_indices /// [`UnicodeSegmentation`]: trait.UnicodeSegmentation.html #[derive(Debug, Clone)] pub struct GraphemeIndices<'a> { start_offset: usize, iter: Graphemes<'a>, } impl<'a> GraphemeIndices<'a> { #[inline] /// View the underlying data (the part yet to be iterated) as a slice of the original string. /// /// ```rust /// # use unicode_segmentation::UnicodeSegmentation; /// let mut iter = "abc".grapheme_indices(true); /// assert_eq!(iter.as_str(), "abc"); /// iter.next(); /// assert_eq!(iter.as_str(), "bc"); /// iter.next(); /// iter.next(); /// assert_eq!(iter.as_str(), ""); /// ``` pub fn as_str(&self) -> &'a str { self.iter.as_str() } } impl<'a> Iterator for GraphemeIndices<'a> { type Item = (usize, &'a str); #[inline] fn next(&mut self) -> Option<(usize, &'a str)> { self.iter .next() .map(|s| (s.as_ptr() as usize - self.start_offset, s)) } #[inline] fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } impl<'a> DoubleEndedIterator for GraphemeIndices<'a> { #[inline] fn next_back(&mut self) -> Option<(usize, &'a str)> { self.iter .next_back() .map(|s| (s.as_ptr() as usize - self.start_offset, s)) } } /// External iterator for a string's /// [grapheme clusters](http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries). /// /// This struct is created by the [`graphemes`] method on the [`UnicodeSegmentation`] trait. See its /// documentation for more. /// /// [`graphemes`]: trait.UnicodeSegmentation.html#tymethod.graphemes /// [`UnicodeSegmentation`]: trait.UnicodeSegmentation.html #[derive(Clone, Debug)] pub struct Graphemes<'a> { string: &'a str, cursor: GraphemeCursor, cursor_back: GraphemeCursor, } impl<'a> Graphemes<'a> { #[inline] /// View the underlying data (the part yet to be iterated) as a slice of the original string. /// /// ```rust /// # use unicode_segmentation::UnicodeSegmentation; /// let mut iter = "abc".graphemes(true); /// assert_eq!(iter.as_str(), "abc"); /// iter.next(); /// assert_eq!(iter.as_str(), "bc"); /// iter.next(); /// iter.next(); /// assert_eq!(iter.as_str(), ""); /// ``` pub fn as_str(&self) -> &'a str { &self.string[self.cursor.cur_cursor()..self.cursor_back.cur_cursor()] } } impl<'a> Iterator for Graphemes<'a> { type Item = &'a str; #[inline] fn size_hint(&self) -> (usize, Option) { let slen = self.cursor_back.cur_cursor() - self.cursor.cur_cursor(); (cmp::min(slen, 1), Some(slen)) } #[inline] fn next(&mut self) -> Option<&'a str> { let start = self.cursor.cur_cursor(); if start == self.cursor_back.cur_cursor() { return None; } let next = self.cursor.next_boundary(self.string, 0).unwrap().unwrap(); Some(&self.string[start..next]) } } impl<'a> DoubleEndedIterator for Graphemes<'a> { #[inline] fn next_back(&mut self) -> Option<&'a str> { let end = self.cursor_back.cur_cursor(); if end == self.cursor.cur_cursor() { return None; } let prev = self .cursor_back .prev_boundary(self.string, 0) .unwrap() .unwrap(); Some(&self.string[prev..end]) } } #[inline] pub fn new_graphemes(s: &str, is_extended: bool) -> Graphemes<'_> { let len = s.len(); Graphemes { string: s, cursor: GraphemeCursor::new(0, len, is_extended), cursor_back: GraphemeCursor::new(len, len, is_extended), } } #[inline] pub fn new_grapheme_indices(s: &str, is_extended: bool) -> GraphemeIndices<'_> { GraphemeIndices { start_offset: s.as_ptr() as usize, iter: new_graphemes(s, is_extended), } } /// maybe unify with PairResult? /// An enum describing information about a potential boundary. #[derive(PartialEq, Eq, Clone, Debug)] enum GraphemeState { /// No information is known. Unknown, /// It is known to not be a boundary. NotBreak, /// It is known to be a boundary. Break, /// The codepoint after it has Indic_Conjunct_Break=Consonant, /// so there is a break before it unless preceded (after skipping /// zero or more InCB=Extend) by an InCB=Linker. (GB9c, Unicode 18) InCbConsonant, /// The codepoint after is a Regional Indicator Symbol, so a boundary iff /// it is preceded by an even number of RIS codepoints. (GB12, GB13) Regional, /// The codepoint after is Extended_Pictographic, /// so whether it's a boundary depends on pre-context according to GB11. Emoji { /// Whether the ZWJ char has been seen already an only a "\p{Extended_Pictographic} Extend*" /// part of GB11 has to be checked seen_zwj: bool, }, } /// Cursor-based segmenter for grapheme clusters. /// /// This allows working with ropes and other datastructures where the string is not contiguous or /// fully known at initialization time. #[derive(Clone, Debug)] pub struct GraphemeCursor { /// Current cursor position. offset: usize, /// Total length of the string. len: usize, /// A config flag indicating whether this cursor computes legacy or extended /// grapheme cluster boundaries (enables GB9a and GB9b if set). is_extended: bool, /// Information about the potential boundary at `offset` state: GraphemeState, /// Category of codepoint immediately preceding cursor, if known. cat_before: Option, /// Category of codepoint immediately after cursor, if known. cat_after: Option, /// If set, at least one more codepoint immediately preceding this offset /// is needed to resolve whether there's a boundary at `offset`. pre_context_offset: Option, /// The number of `InCB=Linker` codepoints preceding `offset` /// (potentially intermingled with `InCB=Extend`). incb_linker_count: Option, /// The number of RIS codepoints preceding `offset`. If `pre_context_offset` /// is set, then counts the number of RIS between that and `offset`, otherwise /// is an accurate count relative to the string. ris_count: Option, /// Set if a call to `prev_boundary` or `next_boundary` was suspended due /// to needing more input. resuming: bool, /// Cached grapheme category and associated scalar value range. grapheme_cat_cache: (u32, u32, GraphemeCat), } /// An error return indicating that not enough content was available in the /// provided chunk to satisfy the query, and that more content must be provided. #[derive(PartialEq, Eq, Debug)] pub enum GraphemeIncomplete { /// More pre-context is needed. The caller should call `provide_context` /// with a chunk ending at the offset given, then retry the query. This /// will only be returned if the `chunk_start` parameter is nonzero. PreContext(usize), /// When requesting `prev_boundary`, the cursor is moving past the beginning /// of the current chunk, so the chunk before that is requested. This will /// only be returned if the `chunk_start` parameter is nonzero. PrevChunk, /// When requesting `next_boundary`, the cursor is moving past the end of the /// current chunk, so the chunk after that is requested. This will only be /// returned if the chunk ends before the `len` parameter provided on /// creation of the cursor. NextChunk, // requesting chunk following the one given /// An error returned when the chunk given does not contain the cursor position. InvalidOffset, } // An enum describing the result from lookup of a pair of categories. #[derive(PartialEq, Eq)] enum PairResult { /// definitely not a break NotBreak, /// definitely a break Break, /// a break iff not in extended mode Extended, /// a break unless in extended mode and preceded by /// an InCB=Linker (possibly followed by InCB=Extend characters). /// (GB9c, Unicode 18) InCbConsonant, /// a break if preceded by an even number of RIS Regional, /// a break if preceded by emoji base and (Extend)* Emoji, } #[inline] fn check_pair(before: GraphemeCat, after: GraphemeCat) -> PairResult { use self::PairResult::*; use crate::tables::grapheme::GraphemeCat::*; match (before, after) { (GC_CR, GC_LF) => NotBreak, // GB3 (GC_Control | GC_CR | GC_LF, _) => Break, // GB4 (_, GC_Control | GC_CR | GC_LF) => Break, // GB5 (GC_L, GC_L | GC_V | GC_LV | GC_LVT) => NotBreak, // GB6 (GC_LV | GC_V, GC_V | GC_T) => NotBreak, // GB7 (GC_LVT | GC_T, GC_T) => NotBreak, // GB8 (_, GC_Extend | GC_ZWJ) => NotBreak, // GB9 (_, GC_SpacingMark) => Extended, // GB9a (GC_Prepend, _) => Extended, // GB9b (_, GC_InCB_Consonant) => InCbConsonant, // GB9c (GC_ZWJ, GC_Extended_Pictographic) => Emoji, // GB11 (GC_Regional_Indicator, GC_Regional_Indicator) => Regional, // GB12, GB13 (_, _) => Break, // GB999 } } /// Whether `ch`, whose grapheme category is `cat`, has `Indic_Conjunct_Break=Extend`. /// /// `InCB=Extend` is defined as /// `[\p{gcb=Extend} \p{gcb=ZWJ}] - \p{InCB=Linker} - \p{InCB=Consonant} - U+200C`, /// and no `InCB=Consonant` is `gcb=Extend` or `gcb=ZWJ`, /// so the grapheme category the caller already has, plus two equality tests, decides it. /// /// Note: since Unicode 18, some `InCB=Linker` characters (e.g. U+1CF5, U+1CF6, U+11A3A) /// have gcb categories other than `Extend` or `ZWJ`. This does not affect correctness: /// those characters are not in `InCB=Extend` either, and `is_incb_extend` correctly /// returns false for them because `may_be_incb` returns false for their gcb category. /// The callers that need to handle `InCB=Linker` (e.g. GB9c) must check /// `is_incb_linker` separately. /// /// `scripts/unicode.py` checks the derivation against the UCD when it regenerates `src/tables.rs`, /// so a future Unicode version cannot silently invalidate it. #[inline] fn is_incb_extend(cat: GraphemeCat, ch: char) -> bool { // ZWNJ is `gcb=Extend` but `InCB=None`. may_be_incb(cat) && ch != '\u{200c}' && !crate::tables::is_incb_linker(ch) } /// `InCB=Extend` is a subset of `gcb=Extend` and `gcb=ZWJ`, so any category /// other than those rules out the `InCB=Extend` role without inspecting the codepoint. /// However, `InCB=Linker` is *not* necessarily a subset of those categories (since /// Unicode 18), so callers that handle Indic conjunct break must check /// `is_incb_linker` independently of this function. #[inline] fn may_be_incb(cat: GraphemeCat) -> bool { matches!(cat, GraphemeCat::GC_Extend | GraphemeCat::GC_ZWJ) } impl GraphemeCursor { /// Create a new cursor. The string and initial offset are given at creation /// time, but the contents of the string are not. The `is_extended` parameter /// controls whether extended grapheme clusters are selected. /// /// The `offset` parameter must be on a codepoint boundary. /// /// ```rust /// # use unicode_segmentation::GraphemeCursor; /// let s = "हिन्दी"; /// let mut legacy = GraphemeCursor::new(0, s.len(), false); /// assert_eq!(legacy.next_boundary(s, 0), Ok(Some("ह".len()))); /// let mut extended = GraphemeCursor::new(0, s.len(), true); /// assert_eq!(extended.next_boundary(s, 0), Ok(Some("हि".len()))); /// ``` pub fn new(offset: usize, len: usize, is_extended: bool) -> GraphemeCursor { let state = if offset == 0 || offset == len { GraphemeState::Break } else { GraphemeState::Unknown }; GraphemeCursor { offset, len, state, is_extended, cat_before: None, cat_after: None, pre_context_offset: None, incb_linker_count: None, ris_count: None, resuming: false, grapheme_cat_cache: (0, 0, GraphemeCat::GC_Control), } } fn grapheme_category(&mut self, ch: char) -> GraphemeCat { use crate::tables::grapheme as gr; use crate::tables::grapheme::GraphemeCat::*; if ch <= '\u{7e}' { // Special-case optimization for ascii, except U+007F. This // improves performance even for many primarily non-ascii texts, // due to use of punctuation and white space characters from the // ascii range. if ch >= '\u{20}' { GC_Any } else if ch == '\n' { GC_LF } else if ch == '\r' { GC_CR } else { GC_Control } } else { // If this char isn't within the cached range, update the cache to the // range that includes it. if (ch as u32) < self.grapheme_cat_cache.0 || (ch as u32) > self.grapheme_cat_cache.1 { self.grapheme_cat_cache = gr::grapheme_category(ch); } self.grapheme_cat_cache.2 } } // Not sure I'm gonna keep this, the advantage over new() seems thin. /// Set the cursor to a new location in the same string. /// /// ```rust /// # use unicode_segmentation::GraphemeCursor; /// let s = "abcd"; /// let mut cursor = GraphemeCursor::new(0, s.len(), false); /// assert_eq!(cursor.cur_cursor(), 0); /// cursor.set_cursor(2); /// assert_eq!(cursor.cur_cursor(), 2); /// ``` pub fn set_cursor(&mut self, offset: usize) { if offset != self.offset { self.offset = offset; self.state = if offset == 0 || offset == self.len { GraphemeState::Break } else { GraphemeState::Unknown }; // reset state derived from text around cursor self.cat_before = None; self.cat_after = None; self.incb_linker_count = None; self.ris_count = None; } } #[inline] /// The current offset of the cursor. Equal to the last value provided to /// `new()` or `set_cursor()`, or returned from `next_boundary()` or /// `prev_boundary()`. /// /// ```rust /// # use unicode_segmentation::GraphemeCursor; /// // Two flags (🇷🇸🇮🇴), each flag is two RIS codepoints, each RIS is 4 bytes. /// let flags = "\u{1F1F7}\u{1F1F8}\u{1F1EE}\u{1F1F4}"; /// let mut cursor = GraphemeCursor::new(4, flags.len(), false); /// assert_eq!(cursor.cur_cursor(), 4); /// assert_eq!(cursor.next_boundary(flags, 0), Ok(Some(8))); /// assert_eq!(cursor.cur_cursor(), 8); /// ``` pub fn cur_cursor(&self) -> usize { self.offset } /// Provide additional pre-context when it is needed to decide a boundary. /// The end of the chunk must coincide with the value given in the /// `GraphemeIncomplete::PreContext` request. /// /// ```rust /// # use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete}; /// let flags = "\u{1F1F7}\u{1F1F8}\u{1F1EE}\u{1F1F4}"; /// let mut cursor = GraphemeCursor::new(8, flags.len(), false); /// // Not enough pre-context to decide if there's a boundary between the two flags. /// assert_eq!(cursor.is_boundary(&flags[8..], 8), Err(GraphemeIncomplete::PreContext(8))); /// // Provide one more Regional Indicator Symbol of pre-context /// cursor.provide_context(&flags[4..8], 4); /// // Still not enough context to decide. /// assert_eq!(cursor.is_boundary(&flags[8..], 8), Err(GraphemeIncomplete::PreContext(4))); /// // Provide additional requested context. /// cursor.provide_context(&flags[0..4], 0); /// // That's enough to decide (it always is when context goes to the start of the string) /// assert_eq!(cursor.is_boundary(&flags[8..], 8), Ok(true)); /// ``` pub fn provide_context(&mut self, chunk: &str, chunk_start: usize) { use crate::tables::grapheme as gr; // Defensive guard: callers are expected to invoke this only in response // to a `GraphemeIncomplete::PreContext` request with matching chunk end. // Ignore mismatched calls instead of panicking (#170). if self.pre_context_offset != Some(chunk_start.saturating_add(chunk.len())) { return; } self.pre_context_offset = None; // GB4 and GB5 come before GB9b, so a Control, CR or LF after the // Prepend still breaks; only apply the GB9b shortcut when the // following category cannot be decided by an earlier rule. let after_breaks_first = matches!(self.cat_after, Some(gr::GC_Control | gr::GC_CR | gr::GC_LF)); if self.is_extended && !after_breaks_first && chunk_start.saturating_add(chunk.len()) == self.offset { let ch = chunk.chars().next_back().unwrap(); if self.grapheme_category(ch) == gr::GC_Prepend { self.decide(false); // GB9b return; } } match self.state { GraphemeState::InCbConsonant => self.handle_incb_consonant(chunk, chunk_start), GraphemeState::Regional => self.handle_regional(chunk, chunk_start), GraphemeState::Emoji { seen_zwj } => self.handle_emoji(chunk, chunk_start, seen_zwj), _ => { if self.cat_before.is_none() && self.offset == chunk.len() + chunk_start { let ch = chunk.chars().next_back().unwrap(); self.cat_before = Some(self.grapheme_category(ch)); } } } } #[inline] fn decide(&mut self, is_break: bool) { self.state = if is_break { GraphemeState::Break } else { GraphemeState::NotBreak }; } #[inline] fn decision(&mut self, is_break: bool) -> Result { self.decide(is_break); Ok(is_break) } #[inline] fn is_boundary_result(&self) -> Result { if self.state == GraphemeState::Break { Ok(true) } else if self.state == GraphemeState::NotBreak { Ok(false) } else if let Some(pre_context_offset) = self.pre_context_offset { Err(GraphemeIncomplete::PreContext(pre_context_offset)) } else { unreachable!("inconsistent state"); } } /// For handling rule GB9c (Unicode 18): /// /// There's an `InCB=Consonant` after this, and we need to look back /// to verify whether there should be a break. /// /// GB9c: `\p{InCB=Linker} \p{InCB=Extend}* × \p{InCB=Consonant}` /// /// Seek backward to find an `InCB=Linker` (potentially preceded by some /// number of `InCB=Extend`). If we find a linker, then there's no break; /// if we find a non-linker non-extend, or the start of text, there's a break; /// otherwise we need more context. #[inline] fn handle_incb_consonant(&mut self, chunk: &str, chunk_start: usize) { use crate::tables::is_incb_linker; // GB9c only applies to extended grapheme clusters if !self.is_extended { self.decide(true); return; } for ch in chunk.chars().rev() { if is_incb_extend(self.grapheme_category(ch), ch) { // Skip InCB extends continue; } else if is_incb_linker(ch) { // Found a linker, suppress break self.decide(false); return; } else { // Not extend, not linker — break self.decide(true); return; } } if chunk_start == 0 { // Start of text and we still haven't found a linker, so break self.decide(true); } else { // We need more context self.pre_context_offset = Some(chunk_start); self.state = GraphemeState::InCbConsonant; } } #[inline] fn handle_regional(&mut self, chunk: &str, chunk_start: usize) { use crate::tables::grapheme as gr; let mut ris_count = self.ris_count.unwrap_or(0); for ch in chunk.chars().rev() { if self.grapheme_category(ch) != gr::GC_Regional_Indicator { self.ris_count = Some(ris_count); self.decide(ris_count % 2 == 0); return; } ris_count += 1; } self.ris_count = Some(ris_count); if chunk_start == 0 { self.decide(ris_count % 2 == 0); } else { self.pre_context_offset = Some(chunk_start); self.state = GraphemeState::Regional; } } #[inline] fn handle_emoji(&mut self, chunk: &str, chunk_start: usize, mut seen_zwj: bool) { // \p{Extended_Pictographic} Extend* ZWJ × \p{Extended_Pictographic} use crate::tables::grapheme as gr; let mut iter = chunk.chars().rev(); if !seen_zwj { if let Some(ch) = iter.next() { if self.grapheme_category(ch) != gr::GC_ZWJ { self.decide(true); return; } else { seen_zwj = true; } } } for ch in iter { match self.grapheme_category(ch) { gr::GC_Extend => (), gr::GC_Extended_Pictographic => { self.decide(false); return; } _ => { self.decide(true); return; } } } if chunk_start == 0 { self.decide(true); } else { self.pre_context_offset = Some(chunk_start); self.state = GraphemeState::Emoji { seen_zwj }; } } #[inline] /// Determine whether the current cursor location is a grapheme cluster boundary. /// Only a part of the string need be supplied. If `chunk_start` is nonzero or /// the length of `chunk` is not equal to `len` on creation, then this method /// may return `GraphemeIncomplete::PreContext`. The caller should then /// call `provide_context` with the requested chunk, then retry calling this /// method. /// /// For partial chunks, if the cursor is not at the beginning or end of the /// string, the chunk should contain at least the codepoint following the cursor. /// If the string is nonempty, the chunk must be nonempty. /// /// All calls should have consistent chunk contents (ie, if a chunk provides /// content for a given slice, all further chunks covering that slice must have /// the same content for it). /// /// ```rust /// # use unicode_segmentation::GraphemeCursor; /// let flags = "\u{1F1F7}\u{1F1F8}\u{1F1EE}\u{1F1F4}"; /// let mut cursor = GraphemeCursor::new(8, flags.len(), false); /// assert_eq!(cursor.is_boundary(flags, 0), Ok(true)); /// cursor.set_cursor(12); /// assert_eq!(cursor.is_boundary(flags, 0), Ok(false)); /// ``` pub fn is_boundary( &mut self, chunk: &str, chunk_start: usize, ) -> Result { use crate::tables::grapheme as gr; if self.state == GraphemeState::Break { return Ok(true); } if self.state == GraphemeState::NotBreak { return Ok(false); } if (self.offset < chunk_start || self.offset >= chunk_start.saturating_add(chunk.len())) && (self.offset > chunk_start.saturating_add(chunk.len()) || self.cat_after.is_none()) { return Err(GraphemeIncomplete::InvalidOffset); } if let Some(pre_context_offset) = self.pre_context_offset { return Err(GraphemeIncomplete::PreContext(pre_context_offset)); } let offset_in_chunk = self.offset.saturating_sub(chunk_start); if self.cat_after.is_none() { let ch = chunk[offset_in_chunk..].chars().next().unwrap(); self.cat_after = Some(self.grapheme_category(ch)); } if self.offset == chunk_start { let mut need_pre_context = true; match self.cat_after.unwrap() { gr::GC_InCB_Consonant => self.state = GraphemeState::InCbConsonant, // Only look back for the RI count if it isn't known already. gr::GC_Regional_Indicator if self.ris_count.is_none() => { self.state = GraphemeState::Regional } gr::GC_Extended_Pictographic => { self.state = GraphemeState::Emoji { seen_zwj: false } } _ => need_pre_context = self.cat_before.is_none(), } if need_pre_context { self.pre_context_offset = Some(chunk_start); return Err(GraphemeIncomplete::PreContext(chunk_start)); } } if self.cat_before.is_none() { let ch = chunk[..offset_in_chunk].chars().next_back().unwrap(); self.cat_before = Some(self.grapheme_category(ch)); } match check_pair(self.cat_before.unwrap(), self.cat_after.unwrap()) { PairResult::NotBreak => self.decision(false), PairResult::Break => self.decision(true), PairResult::Extended => { let is_extended = self.is_extended; self.decision(!is_extended) } PairResult::InCbConsonant => { self.handle_incb_consonant(&chunk[..offset_in_chunk], chunk_start); self.is_boundary_result() } PairResult::Regional => { if let Some(ris_count) = self.ris_count { return self.decision((ris_count % 2) == 0); } self.handle_regional(&chunk[..offset_in_chunk], chunk_start); self.is_boundary_result() } PairResult::Emoji => { self.handle_emoji(&chunk[..offset_in_chunk], chunk_start, false); self.is_boundary_result() } } } #[inline] /// Find the next boundary after the current cursor position. Only a part of /// the string need be supplied. If the chunk is incomplete, then this /// method might return `GraphemeIncomplete::PreContext` or /// `GraphemeIncomplete::NextChunk`. In the former case, the caller should /// call `provide_context` with the requested chunk, then retry. In the /// latter case, the caller should provide the chunk following the one /// given, then retry. /// /// See `is_boundary` for expectations on the provided chunk. /// /// ```rust /// # use unicode_segmentation::GraphemeCursor; /// let flags = "\u{1F1F7}\u{1F1F8}\u{1F1EE}\u{1F1F4}"; /// let mut cursor = GraphemeCursor::new(4, flags.len(), false); /// assert_eq!(cursor.next_boundary(flags, 0), Ok(Some(8))); /// assert_eq!(cursor.next_boundary(flags, 0), Ok(Some(16))); /// assert_eq!(cursor.next_boundary(flags, 0), Ok(None)); /// ``` /// /// And an example that uses partial strings: /// /// ```rust /// # use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete}; /// let s = "abcd"; /// let mut cursor = GraphemeCursor::new(0, s.len(), false); /// assert_eq!(cursor.next_boundary(&s[..2], 0), Ok(Some(1))); /// assert_eq!(cursor.next_boundary(&s[..2], 0), Err(GraphemeIncomplete::NextChunk)); /// assert_eq!(cursor.next_boundary(&s[2..4], 2), Ok(Some(2))); /// assert_eq!(cursor.next_boundary(&s[2..4], 2), Ok(Some(3))); /// assert_eq!(cursor.next_boundary(&s[2..4], 2), Ok(Some(4))); /// assert_eq!(cursor.next_boundary(&s[2..4], 2), Ok(None)); /// ``` pub fn next_boundary( &mut self, chunk: &str, chunk_start: usize, ) -> Result, GraphemeIncomplete> { if self.offset == self.len { return Ok(None); } // Defensive guard: the cursor must stay within the text. Out-of-range // cursors used to slice out of bounds or unwrap an empty iterator (#170). if self.offset > self.len || chunk_start > self.offset { return Err(GraphemeIncomplete::InvalidOffset); } let mut iter = chunk[self.offset.saturating_sub(chunk_start)..].chars(); let mut ch = match iter.next() { Some(ch) => ch, None => return Err(GraphemeIncomplete::NextChunk), }; loop { if self.resuming { if self.cat_after.is_none() { self.cat_after = Some(self.grapheme_category(ch)); } } else { self.offset = self.offset.saturating_add(ch.len_utf8()); self.state = GraphemeState::Unknown; self.cat_before = self.cat_after.take(); if self.cat_before.is_none() { self.cat_before = Some(self.grapheme_category(ch)); } // ZWNJ is the one `gcb=Extend` that is `InCB=None`. // Since Unicode 18, some InCB=Linker characters have gcb categories // other than Extend or ZWJ, so check is_incb_linker first. if ch == '\u{200c}' { self.incb_linker_count = Some(0); } else if crate::tables::is_incb_linker(ch) { self.incb_linker_count = Some(self.incb_linker_count.map_or(1, |c| c + 1)); } else if !may_be_incb(self.cat_before.unwrap()) { self.incb_linker_count = Some(0); } if self.cat_before.unwrap() == GraphemeCat::GC_Regional_Indicator { self.ris_count = self.ris_count.map(|c| c + 1); } else { self.ris_count = Some(0); } if let Some(next_ch) = iter.next() { ch = next_ch; self.cat_after = Some(self.grapheme_category(ch)); } else if self.offset == self.len { self.decide(true); } else { self.resuming = true; return Err(GraphemeIncomplete::NextChunk); } } self.resuming = true; if self.is_boundary(chunk, chunk_start)? { self.resuming = false; return Ok(Some(self.offset)); } self.resuming = false; } } /// Find the previous boundary after the current cursor position. Only a part /// of the string need be supplied. If the chunk is incomplete, then this /// method might return `GraphemeIncomplete::PreContext` or /// `GraphemeIncomplete::PrevChunk`. In the former case, the caller should /// call `provide_context` with the requested chunk, then retry. In the /// latter case, the caller should provide the chunk preceding the one /// given, then retry. /// /// See `is_boundary` for expectations on the provided chunk. /// /// ```rust /// # use unicode_segmentation::GraphemeCursor; /// let flags = "\u{1F1F7}\u{1F1F8}\u{1F1EE}\u{1F1F4}"; /// let mut cursor = GraphemeCursor::new(12, flags.len(), false); /// assert_eq!(cursor.prev_boundary(flags, 0), Ok(Some(8))); /// assert_eq!(cursor.prev_boundary(flags, 0), Ok(Some(0))); /// assert_eq!(cursor.prev_boundary(flags, 0), Ok(None)); /// ``` /// /// And an example that uses partial strings (note the exact return is not /// guaranteed, and may be `PrevChunk` or `PreContext` arbitrarily): /// /// ```rust /// # use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete}; /// let s = "abcd"; /// let mut cursor = GraphemeCursor::new(4, s.len(), false); /// assert_eq!(cursor.prev_boundary(&s[2..4], 2), Ok(Some(3))); /// assert_eq!(cursor.prev_boundary(&s[2..4], 2), Err(GraphemeIncomplete::PrevChunk)); /// assert_eq!(cursor.prev_boundary(&s[0..2], 0), Ok(Some(2))); /// assert_eq!(cursor.prev_boundary(&s[0..2], 0), Ok(Some(1))); /// assert_eq!(cursor.prev_boundary(&s[0..2], 0), Ok(Some(0))); /// assert_eq!(cursor.prev_boundary(&s[0..2], 0), Ok(None)); /// ``` pub fn prev_boundary( &mut self, chunk: &str, chunk_start: usize, ) -> Result, GraphemeIncomplete> { if self.offset == 0 { return Ok(None); } // Defensive guard: the cursor must stay within the text, and the chunk // must start at or before it. Out-of-range cursors used to slice out // of bounds or unwrap an empty iterator (#170). if self.offset > self.len || chunk_start > self.offset { return Err(GraphemeIncomplete::InvalidOffset); } if self.offset == chunk_start { return Err(GraphemeIncomplete::PrevChunk); } // Clamp to the chunk to avoid panicking on out-of-range offsets // coming from a desynchronized or fuzzed caller (#119); the loop // below then reports `PrevChunk` as for any chunk without a // preceding boundary. let end = self.offset.saturating_sub(chunk_start).min(chunk.len()); let mut iter = chunk[..end].chars().rev(); let mut ch = iter.next().unwrap(); loop { if self.offset == chunk_start { self.resuming = true; return Err(GraphemeIncomplete::PrevChunk); } if self.resuming { self.cat_before = Some(self.grapheme_category(ch)); } else { self.offset -= ch.len_utf8(); self.cat_after = self.cat_before.take(); self.state = GraphemeState::Unknown; if let Some(incb_linker_count) = self.incb_linker_count { self.incb_linker_count = if crate::tables::is_incb_linker(ch) { if incb_linker_count > 0 { Some(incb_linker_count - 1) } else { Some(0) } } else if is_incb_extend(self.grapheme_category(ch), ch) { Some(incb_linker_count) } else { None }; } if let Some(ris_count) = self.ris_count { self.ris_count = if ris_count > 0 { Some(ris_count - 1) } else { None }; } if let Some(prev_ch) = iter.next() { ch = prev_ch; self.cat_before = Some(self.grapheme_category(ch)); } else if self.offset == 0 { self.decide(true); } else { self.resuming = true; self.cat_after = Some(self.grapheme_category(ch)); return Err(GraphemeIncomplete::PrevChunk); } } self.resuming = true; if self.is_boundary(chunk, chunk_start)? { self.resuming = false; return Ok(Some(self.offset)); } self.resuming = false; } } } #[test] fn test_grapheme_cursor_ris_count_across_chunks() { use GraphemeIncomplete::*; let chunk0 = "a"; // 1 byte let chunk1 = "\u{1f1e6}"; // 4 bytes let chunk2 = "\u{1f1e6}"; // 4 bytes let full_len = chunk0.len() + chunk1.len() + chunk2.len(); // 9 let chunk1_start = chunk0.len(); let chunk2_start = chunk0.len() + chunk1.len(); let mut c = GraphemeCursor::new(0, full_len, true); assert_eq!(c.next_boundary(chunk0, 0), Err(NextChunk)); assert_eq!(c.next_boundary(chunk1, chunk1_start), Ok(Some(1))); assert_eq!(c.next_boundary(chunk1, chunk1_start), Err(NextChunk)); assert_eq!(c.next_boundary(chunk2, chunk2_start), Ok(Some(9))); assert_eq!(c.next_boundary(chunk2, chunk2_start), Ok(None)); } #[test] fn test_grapheme_cursor_ris_precontext() { let s = "\u{1f1fa}\u{1f1f8}\u{1f1fa}\u{1f1f8}\u{1f1fa}\u{1f1f8}"; let mut c = GraphemeCursor::new(8, s.len(), true); assert_eq!( c.is_boundary(&s[4..], 4), Err(GraphemeIncomplete::PreContext(4)) ); c.provide_context(&s[..4], 0); assert_eq!(c.is_boundary(&s[4..], 4), Ok(true)); } #[test] fn test_grapheme_cursor_chunk_start_require_precontext() { let s = "\r\n"; let mut c = GraphemeCursor::new(1, s.len(), true); assert_eq!( c.is_boundary(&s[1..], 1), Err(GraphemeIncomplete::PreContext(1)) ); c.provide_context(&s[..1], 0); assert_eq!(c.is_boundary(&s[1..], 1), Ok(false)); } #[test] fn test_grapheme_cursor_prepend_before_control_on_chunk_start() { // GB4 and GB5 take precedence over GB9b, so `Prepend` does not glue a // following `Control`, `CR` or `LF` to itself. GraphemeBreakTest.txt: // ÷ 06DD ÷ 000D ÷ # ÷ [0.2] ARABIC END OF AYAH (Prepend) ÷ [5.0] (CR) ÷ [0.3] for s in ["\u{06dd}\r", "\u{06dd}\n", "\u{06dd}\0"] { let prepend_len = "\u{06dd}".len(); // The whole-string answer, which the `Graphemes` iterator already gets right. assert_eq!( GraphemeCursor::new(prepend_len, s.len(), true).is_boundary(s, 0), Ok(true) ); // The same query answered from separate chunks must agree. let mut c = GraphemeCursor::new(prepend_len, s.len(), true); assert_eq!( c.is_boundary(&s[prepend_len..], prepend_len), Err(GraphemeIncomplete::PreContext(prepend_len)) ); c.provide_context(&s[..prepend_len], 0); assert_eq!(c.is_boundary(&s[prepend_len..], prepend_len), Ok(true)); } } #[test] fn test_grapheme_cursor_prev_boundary() { let s = "abcd"; let mut c = GraphemeCursor::new(3, s.len(), true); assert_eq!( c.prev_boundary(&s[2..], 2), Err(GraphemeIncomplete::PrevChunk) ); assert_eq!(c.prev_boundary(&s[..2], 0), Ok(Some(2))); } #[test] fn test_grapheme_cursor_prev_boundary_chunk_start() { let s = "abcd"; let mut c = GraphemeCursor::new(2, s.len(), true); assert_eq!( c.prev_boundary(&s[2..], 2), Err(GraphemeIncomplete::PrevChunk) ); assert_eq!(c.prev_boundary(&s[..2], 0), Ok(Some(1))); } #[test] fn test_grapheme_cursor_boundary_with_zwj_on_chunk_start() { use GraphemeIncomplete::*; let chunk0 = "👩"; // 4 bytes let chunk1 = "\u{200d}🔬"; // 3 bytes + 4 bytes let full_len = chunk0.len() + chunk1.len(); let mut cur = GraphemeCursor::new(0, full_len, true); assert_eq!(cur.next_boundary(chunk0, 0), Err(NextChunk)); match cur.next_boundary(chunk1, chunk0.len()) { Ok(res) => assert_eq!(res, Some(11)), Err(PreContext(_)) => { cur.provide_context(chunk0, 0); assert_eq!(cur.next_boundary(chunk1, chunk0.len()), Ok(Some(11))); } _ => unreachable!(), } } #[test] fn test_grapheme_cursor_emoji_no_zwj() { use GraphemeIncomplete::*; let chunk0 = "🍒"; // 4 bytes let chunk1 = "🥑"; // 4 bytes let full_len = chunk0.len() + chunk1.len(); let mut c = GraphemeCursor::new(0, full_len, true); assert_eq!(c.next_boundary(chunk0, 0), Err(NextChunk)); assert_eq!( c.next_boundary(chunk1, chunk0.len()), Err(PreContext(chunk0.len())) ); c.provide_context(chunk0, 0); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(Some(4))); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(Some(8))); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(None)); } #[test] fn test_grapheme_cursor_emoji_chunk_boundary_before_zwj() { use GraphemeIncomplete::*; let chunk0 = "🍒"; // 4 bytes let chunk1 = "\u{200d}🥑"; // 3 + 4 bytes let full_len = chunk0.len() + chunk1.len(); // 11 let mut c = GraphemeCursor::new(0, full_len, true); assert_eq!(c.next_boundary(chunk0, 0), Err(NextChunk)); assert_eq!( c.next_boundary(chunk1, chunk0.len()), Err(PreContext(chunk0.len())) ); c.provide_context(chunk0, 0); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(Some(11))); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(None)); } #[test] fn test_grapheme_cursor_emoji_chunk_boundary_after_zwj() { use GraphemeIncomplete::*; let chunk0 = "🍒\u{200d}"; // 4 + 3 bytes let chunk1 = "🥑"; // 4 bytes let full_len = chunk0.len() + chunk1.len(); // 11 let mut c = GraphemeCursor::new(0, full_len, true); assert_eq!(c.next_boundary(chunk0, 0), Err(NextChunk)); assert_eq!( c.next_boundary(chunk1, chunk0.len()), Err(PreContext(chunk0.len())) ); c.provide_context(chunk0, 0); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(Some(11))); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(None)); } #[test] fn test_grapheme_cursor_emoji_zwj_across_chunks() { use GraphemeIncomplete::*; let chunk0 = "🍒"; // 4 bytes let chunk1 = "\u{200d}"; // 3 bytes let chunk2 = "🥑"; // 4 bytes let full_len = chunk0.len() + chunk1.len() + chunk2.len(); // 11 let chunk2_start = chunk0.len() + chunk1.len(); let mut c = GraphemeCursor::new(0, full_len, true); assert_eq!(c.next_boundary(chunk0, 0), Err(NextChunk)); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Err(NextChunk)); assert_eq!( c.next_boundary(chunk2, chunk2_start), Err(PreContext(chunk2_start)) ); c.provide_context(chunk1, chunk0.len()); assert_eq!( c.next_boundary(chunk2, chunk2_start), Err(PreContext(chunk0.len())) ); c.provide_context(chunk0, 0); assert_eq!(c.next_boundary(chunk2, chunk2_start), Ok(Some(11))); assert_eq!(c.next_boundary(chunk2, chunk2_start), Ok(None)); } #[test] fn test_incb_consonant_basic() { // GB9c (Unicode 18): \p{InCB=Linker} \p{InCB=Extend}* × \p{InCB=Consonant} // U+094D = Devanagari Sign Virama (InCB=Linker, gcb=Extend) // U+0915 = Devanagari Letter Ka (InCB=Consonant, gcb=InCB_Consonant) // U+093C = Devanagari Sign Nukta (gcb=Extend) // Linker + Consonant → no break (extended), break (legacy) let s = "\u{94d}\u{915}"; let mut c = GraphemeCursor::new(0, s.len(), true); assert_eq!(c.next_boundary(s, 0), Ok(Some(s.len()))); let mut c = GraphemeCursor::new(0, s.len(), false); assert_eq!(c.next_boundary(s, 0), Ok(Some(3))); // Linker + Extend + Consonant → no break (extended) // U+093C is GC_Extend, so the extend is skipped and the linker is found let s = "\u{94d}\u{93c}\u{915}"; let mut c = GraphemeCursor::new(0, s.len(), true); assert_eq!(c.next_boundary(s, 0), Ok(Some(s.len()))); } #[test] fn test_incb_consonant_no_linker() { // Non-linker + Consonant → break in both modes // U+0903 = Devanagari Sign Anusvara (gcb=SpacingMark) // U+0915 = Devanagari Letter Ka (gcb=InCB_Consonant) let s = "\u{903}\u{915}"; let mut c = GraphemeCursor::new(0, s.len(), true); assert_eq!(c.next_boundary(s, 0), Ok(Some(3))); let mut c = GraphemeCursor::new(0, s.len(), false); assert_eq!(c.next_boundary(s, 0), Ok(Some(3))); } #[test] fn test_grapheme_cursor_incb_consonant_across_chunks() { use GraphemeIncomplete::*; // U+094D = Virama (Linker, 3 bytes), U+0915 = Ka (Consonant, 3 bytes) // Split so the boundary falls between linker and consonant. let chunk0 = "\u{94d}"; // 3 bytes let chunk1 = "\u{915}"; // 3 bytes let full_len = chunk0.len() + chunk1.len(); let mut c = GraphemeCursor::new(0, full_len, true); assert_eq!(c.next_boundary(chunk0, 0), Err(NextChunk)); // The consonant triggers GB9c lookback; linker is in a previous chunk. assert_eq!( c.next_boundary(chunk1, chunk0.len()), Err(PreContext(chunk0.len())) ); c.provide_context(chunk0, 0); assert_eq!(c.next_boundary(chunk1, chunk0.len()), Ok(Some(full_len))); } #[test] fn test_incb_extend_excludes_linkers() { // U+1CF5 and U+1CF6 are InCB=Linker, not InCB=Extend. // is_incb_linker must return true for them. assert!(!crate::tables::is_incb_linker('\u{200c}')); // ZWNJ is not a linker assert!(crate::tables::is_incb_linker('\u{94d}')); // Virama is a linker assert!(crate::tables::is_incb_linker('\u{1cf5}')); // Vedic Jihvamuliya (Unicode 18 linker) assert!(crate::tables::is_incb_linker('\u{1cf6}')); // Vedic Upadhmaniya (Unicode 18 linker) assert!(crate::tables::is_incb_linker('\u{11a3a}')); // Zanabazar Square (Unicode 18 linker) }