// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using Microsoft.Recognizers.Text.Matcher; namespace Microsoft.Recognizers.Text.NumberWithUnit { public class BaseMergedUnitExtractor : IExtractor { private readonly INumberWithUnitExtractorConfiguration config; private readonly NumberWithUnitExtractor numberWithUnitExtractor; public BaseMergedUnitExtractor(INumberWithUnitExtractorConfiguration config) { this.config = config; this.numberWithUnitExtractor = new NumberWithUnitExtractor(config); } public List Extract(string source) { List ers; // Only merge currency's compound units for now. if (config.ExtractType.Equals(Constants.SYS_UNIT_CURRENCY, StringComparison.Ordinal)) { ers = MergeCompoundUnits(source); } else { ers = numberWithUnitExtractor.Extract(source); } return ers; } private List MergeCompoundUnits(string source) { var result = new List(); var ers = numberWithUnitExtractor.Extract(source); MergePureNumber(source, ers); if (ers.Count == 0) { return result; } var groups = new int[ers.Count]; groups[0] = 0; for (var idx = 0; idx < ers.Count - 1; idx++) { if (ers[idx].Type != ers[idx + 1].Type && !ers[idx].Type.Equals(Constants.SYS_NUM, StringComparison.Ordinal) && !ers[idx + 1].Type.Equals(Constants.SYS_NUM, StringComparison.Ordinal)) { continue; } if (ers[idx].Data is ExtractResult er && !er.Data.ToString().StartsWith(Number.Constants.INTEGER_PREFIX, StringComparison.Ordinal)) { groups[idx + 1] = groups[idx] + 1; continue; } var middleBegin = ers[idx].Start + ers[idx].Length ?? 0; var middleEnd = ers[idx + 1].Start ?? 0; var length = middleEnd - middleBegin; if (length < 0) { continue; } var middleStr = source.Substring(middleBegin, length).Trim(); // Separated by whitespace if (string.IsNullOrEmpty(middleStr)) { groups[idx + 1] = groups[idx]; continue; } // Separated by connectors var match = config.CompoundUnitConnectorRegex.Match(middleStr); if (match.Success && match.Index == 0 && match.Length == middleStr.Length) { groups[idx + 1] = groups[idx]; } else { groups[idx + 1] = groups[idx] + 1; } } for (var idx = 0; idx < ers.Count; idx++) { if (idx == 0 || groups[idx] != groups[idx - 1]) { var tmpExtractResult = ers[idx].Clone(); tmpExtractResult.Data = new List { new ExtractResult { Data = ers[idx].Data, Length = ers[idx].Length, Start = ers[idx].Start, Text = ers[idx].Text, Type = ers[idx].Type, }, }; result.Add(tmpExtractResult); } // Reduce extract results in same group if (idx + 1 < ers.Count && groups[idx + 1] == groups[idx]) { var group = groups[idx]; var periodBegin = result[group].Start ?? 0; var periodEnd = (ers[idx + 1].Start ?? 0) + (ers[idx + 1].Length ?? 0); result[group].Length = periodEnd - periodBegin; result[group].Text = source.Substring(periodBegin, periodEnd - periodBegin); result[group].Type = Constants.SYS_UNIT_CURRENCY; (result[group].Data as List)?.Add(ers[idx + 1]); } } for (var idx = 0; idx < result.Count; idx++) { var innerData = result[idx].Data as List; if (innerData?.Count == 1) { result[idx] = innerData[0]; } } result.RemoveAll(o => o.Type == Constants.SYS_NUM); MergeMultiplier(source, result); return result; } private void MergePureNumber(string source, List ers) { var numErs = config.UnitNumExtractor.Extract(source); var unitNumbers = new List(); for (int i = 0, j = 0; i < numErs.Count; i++) { bool hasBehindExtraction = false; while (j < ers.Count && ers[j].Start + ers[j].Length < numErs[i].Start) { hasBehindExtraction = true; j++; } if (!hasBehindExtraction) { continue; } // Filter cases like "1 dollars 11a", "11" is not the fraction here. if (source.Length > numErs[i].Start + numErs[i].Length) { var endChar = source.Substring(numErs[i].Length + numErs[i].Start ?? 0, 1); if (char.IsLetter(endChar[0]) && !SimpleTokenizer.IsCjk(endChar[0])) { continue; } } var middleBegin = ers[j - 1].Start + ers[j - 1].Length ?? 0; var middleEnd = numErs[i].Start ?? 0; var middleStr = source.Substring(middleBegin, middleEnd - middleBegin).Trim(); // Separated by whitespace if (string.IsNullOrEmpty(middleStr)) { unitNumbers.Add(numErs[i]); continue; } // Separated by connectors var match = config.CompoundUnitConnectorRegex.Match(middleStr); if (match.Success && match.Index == 0 && match.Length == middleStr.Length) { unitNumbers.Add(numErs[i]); } } foreach (var extractResult in unitNumbers) { var overlap = false; foreach (var er in ers) { if (er.Start <= extractResult.Start && er.Start + er.Length >= extractResult.Start) { overlap = true; } } if (!overlap) { ers.Add(extractResult); } } ers.Sort((x, y) => x.Start - y.Start ?? 0); } // Add multiplier to extraction when it follows the unit e.g. "10 USD million" private void MergeMultiplier(string source, List ers) { if (config.MultiplierRegex != null) { var multiplierMatches = config.MultiplierRegex.Matches(source); Match[] multipliers = new Match[multiplierMatches.Count]; multiplierMatches.CopyTo(multipliers, 0); if (multipliers.Length > 0) { for (int i = 0; i < ers.Count; i++) { var afterMatch = multipliers.Where(o => ers[i].Start + ers[i].Length == o.Index).ToList(); if (afterMatch.Count > 0 && ers[i].Data != null) { ers[i].Data = new List { ers[i].Clone() }; ers[i].Length += afterMatch[0].Length; ers[i].Text = source.Substring((int)ers[i].Start, (int)ers[i].Length); ers[i].Metadata = new Metadata { HasMod = true }; } } } } } } }