// SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 /** * Split the text into sentences. */ export const splitSentences = ( text: string, minLength = 20, retainFormat: boolean = false, ): [string, number, number][] => { const alphabets = /([A-Za-z])/g; const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g; const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g; const starters = /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)/g; const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g; const websites = /(\w+\.)+(com|net|org|io|gov|edu|me)/g; const digits = /([0-9])/g; const dots = /\.{2,}/g; if (retainFormat) { text = text.replaceAll('\n', ''); } else { text = text.replaceAll('\n', ' '); } text = text.replaceAll(prefixes, '$1'); text = text.replace(websites, (match) => match.replaceAll('.', '')); text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1$2'); text = text.replaceAll(dots, (match) => ''.repeat(match.length)); text = text.replaceAll('Ph.D.', 'PhD'); text = text.replaceAll(new RegExp(`\\s${alphabets.source}[.] `, 'g'), ' $1 '); text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1 $2'); text = text.replaceAll( new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'), '$1$2$3', ); text = text.replaceAll( new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'), '$1$2', ); text = text.replaceAll( new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'), '$1 $2', ); text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1'); text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1'); text = text.replaceAll('.”', '”.'); text = text.replaceAll('."', '".'); text = text.replaceAll('!"', '"!'); text = text.replaceAll('?"', '"?'); text = text.replace(/\.(?=\s|$)/g, '.'); text = text.replaceAll('?', '?'); text = text.replaceAll('!', '!'); text = text.replaceAll('', '.'); if (retainFormat) { text = text.replaceAll('', '\n'); } const split = text.split(''); text = text.replaceAll('', ''); const sentences: [string, number, number][] = []; let buf = ''; let start = 0; let end = 0; const prePad = retainFormat ? '' : ' '; for (const match of split) { const sentence = retainFormat ? match : match.trim(); if (!sentence) continue; buf += prePad + sentence; end += match.length; if (buf.length > minLength) { sentences.push([buf.slice(prePad.length), start, end]); start = end; buf = ''; } } if (buf) { // the trailing buffer runs to the end of the text — an end offset of `length - 1` // leaves the last character stranded for any caller that slices by these offsets // (see `xmlWrapTokenizer`, which rebuilds sentences from them) sentences.push([buf.slice(prePad.length), start, text.length]); } return sentences; };