// Copyright (c) Microsoft. All rights reserved. using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Google.Apis.CustomSearchAPI.v1; using Google.Apis.Services; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel.Data; namespace Microsoft.SemanticKernel.Plugins.Web.Google; /// /// A Google Text Search implementation that can be used to perform searches using the Google Web Search API. /// #pragma warning disable CS0618 // ITextSearch is obsolete public sealed class GoogleTextSearch : ITextSearch, ITextSearch, IDisposable #pragma warning restore CS0618 { /// /// Initializes a new instance of the class. /// /// Google Search Engine ID (looks like "a12b345...") /// Google Custom Search API (looks like "ABcdEfG1...") /// Options used when creating this instance of . public GoogleTextSearch( string searchEngineId, string apiKey, GoogleTextSearchOptions? options = null) : this(new BaseClientService.Initializer { ApiKey = apiKey }, searchEngineId, options) { Verify.NotNullOrWhiteSpace(apiKey); } /// /// Initializes a new instance of the class. /// /// The connector initializer /// Google Search Engine ID (looks like "a12b345...") /// Options used when creating this instance of . public GoogleTextSearch( BaseClientService.Initializer initializer, string searchEngineId, GoogleTextSearchOptions? options = null) { Verify.NotNull(initializer); Verify.NotNullOrWhiteSpace(searchEngineId); this._search = new CustomSearchAPIService(initializer); this._searchEngineId = searchEngineId; this._logger = options?.LoggerFactory?.CreateLogger(typeof(GoogleTextSearch)) ?? NullLogger.Instance; this._stringMapper = options?.StringMapper ?? s_defaultStringMapper; this._resultMapper = options?.ResultMapper ?? s_defaultResultMapper; this._options = options; } #pragma warning disable CS0618 // Obsolete ITextSearch, TextSearchOptions, TextSearchFilter, FilterClause #region Legacy ITextSearch Implementation /// public async Task> GetSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { searchOptions ??= new TextSearchOptions(); var filters = ExtractFiltersFromLegacy(searchOptions.Filter); var searchResponse = await this.ExecuteSearchAsync(query, searchOptions.Top, searchOptions.Skip, filters, cancellationToken).ConfigureAwait(false); long? totalCount = searchOptions.IncludeTotalCount ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; return new KernelSearchResults(this.GetResultsAsResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } /// public async Task> GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { searchOptions ??= new TextSearchOptions(); var filters = ExtractFiltersFromLegacy(searchOptions.Filter); var searchResponse = await this.ExecuteSearchAsync(query, searchOptions.Top, searchOptions.Skip, filters, cancellationToken).ConfigureAwait(false); long? totalCount = searchOptions.IncludeTotalCount ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } /// public async Task> SearchAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { searchOptions ??= new TextSearchOptions(); var filters = ExtractFiltersFromLegacy(searchOptions.Filter); var searchResponse = await this.ExecuteSearchAsync(query, searchOptions.Top, searchOptions.Skip, filters, cancellationToken).ConfigureAwait(false); long? totalCount = searchOptions.IncludeTotalCount ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } #endregion #pragma warning restore CS0618 #region ITextSearch Implementation /// public async Task> GetSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { var (top, skip, includeTotalCount, filters) = ExtractSearchParameters(searchOptions); var searchResponse = await this.ExecuteSearchAsync(query, top, skip, filters, cancellationToken).ConfigureAwait(false); long? totalCount = includeTotalCount ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; return new KernelSearchResults(this.GetResultsAsGoogleWebPageAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } /// public async Task> GetTextSearchResultsAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { var (top, skip, includeTotalCount, filters) = ExtractSearchParameters(searchOptions); var searchResponse = await this.ExecuteSearchAsync(query, top, skip, filters, cancellationToken).ConfigureAwait(false); long? totalCount = includeTotalCount ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; return new KernelSearchResults(this.GetResultsAsTextSearchResultAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } /// public async Task> SearchAsync(string query, TextSearchOptions? searchOptions = null, CancellationToken cancellationToken = default) { var (top, skip, includeTotalCount, filters) = ExtractSearchParameters(searchOptions); var searchResponse = await this.ExecuteSearchAsync(query, top, skip, filters, cancellationToken).ConfigureAwait(false); long? totalCount = includeTotalCount ? long.Parse(searchResponse.SearchInformation.TotalResults) : null; return new KernelSearchResults(this.GetResultsAsStringAsync(searchResponse, cancellationToken), totalCount, GetResultsMetadata(searchResponse)); } /// /// Extracts search parameters from generic . /// This is the primary entry point for the LINQ-based filtering path. /// private static (int Top, int Skip, bool IncludeTotalCount, List<(string FieldName, object Value)> Filters) ExtractSearchParameters(TextSearchOptions? searchOptions) { var top = searchOptions?.Top ?? 3; var skip = searchOptions?.Skip ?? 0; var includeTotalCount = searchOptions?.IncludeTotalCount ?? false; var filters = searchOptions?.Filter != null ? ExtractFiltersFromExpression(searchOptions.Filter) : []; return (top, skip, includeTotalCount, filters); } /// /// Walks a LINQ expression tree and extracts Google API filter key-value pairs directly. /// Supports property equality expressions, string Contains operations, NOT operations (inequality and negation), /// and compound AND expressions that map to Google's filter capabilities. /// /// The LINQ expression to walk. /// A list of (FieldName, Value) pairs for Google API filters. /// Thrown when the expression cannot be converted to Google filters. private static List<(string FieldName, object Value)> ExtractFiltersFromExpression(Expression> linqExpression) { var filters = new List<(string FieldName, object Value)>(); ProcessFilterNode(linqExpression.Body, filters); return filters; } /// /// Recursively processes a LINQ expression tree node and builds Google API filters. /// Handles AND combinations, equality, inequality, string Contains, and negation patterns. /// /// The expression node to process. /// The filter list to accumulate results into. /// Thrown when the expression cannot be converted to Google filters. private static void ProcessFilterNode(Expression expression, List<(string FieldName, object Value)> filters) { switch (expression) { case BinaryExpression { NodeType: ExpressionType.AndAlso } andExpr: ProcessFilterNode(andExpr.Left, filters); ProcessFilterNode(andExpr.Right, filters); break; case BinaryExpression { NodeType: ExpressionType.Equal, Left: MemberExpression member, Right: ConstantExpression constant } when constant.Value is not null: { var filterName = MapPropertyToGoogleFilter(member.Member.Name) ?? throw new NotSupportedException( $"Property '{member.Member.Name}' is not supported for Google API equality filters. " + $"Supported patterns: {string.Join(", ", s_supportedPatterns)}."); filters.Add((filterName, constant.Value)); break; } case BinaryExpression { NodeType: ExpressionType.NotEqual, Left: MemberExpression member, Right: ConstantExpression constant } when constant.Value is not null: { _ = MapPropertyToGoogleFilter(member.Member.Name) ?? throw new NotSupportedException( $"Property '{member.Member.Name}' is not supported for Google API inequality filters. " + $"Supported patterns: {string.Join(", ", s_supportedPatterns)}."); filters.Add(("excludeTerms", constant.Value)); break; } case MethodCallExpression { Method.Name: "Contains", Method.DeclaringType: Type dt, Object: MemberExpression member } methodCall when dt == typeof(string) && methodCall.Arguments is [ConstantExpression { Value: not null } argExpr]: { var filterName = MapPropertyToGoogleFilter(member.Member.Name) ?? throw new NotSupportedException( $"Property '{member.Member.Name}' is not supported for Google API Contains filters. " + $"Supported patterns: {string.Join(", ", s_supportedPatterns)}."); // For Contains operations on text fields, use orTerms (more flexible than exactTerms) filters.Add((filterName == "exactTerms" ? "orTerms" : filterName, argExpr.Value)); break; } // Null-guard patterns (page.Property != null, page.Property == null) — silently skip case BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual, Right: ConstantExpression { Value: null } }: break; case MethodCallExpression { Method.Name: "Contains", Object: null }: throw new NotSupportedException( "Collection Contains filters (e.g., collection.Contains(page.Property)) are not supported by Google Custom Search API. " + "Google API doesn't support OR logic across multiple values for a single field. " + "Consider using multiple separate queries instead."); case UnaryExpression { NodeType: ExpressionType.Not } unaryExpr: ProcessNegatedFilterNode(unaryExpr.Operand, filters); break; default: throw new NotSupportedException( $"Expression type '{expression.NodeType}' is not supported for Google API filters. " + $"Supported patterns: {string.Join(", ", s_supportedPatterns)}."); } } /// /// Processes negated expressions (! operator), mapping them to Google's excludeTerms parameter. /// Handles both !(property == value) and !property.Contains(value) patterns. /// /// The inner expression of the NOT operation. /// The filter list to accumulate results into. /// Thrown when the negated expression cannot be converted to Google filters. private static void ProcessNegatedFilterNode(Expression expression, List<(string FieldName, object Value)> filters) { switch (expression) { case BinaryExpression { NodeType: ExpressionType.Equal, Left: MemberExpression member, Right: ConstantExpression constant } when constant.Value is not null: { _ = MapPropertyToGoogleFilter(member.Member.Name) ?? throw new NotSupportedException( $"Property '{member.Member.Name}' is not supported for Google API negated equality filters."); filters.Add(("excludeTerms", constant.Value)); break; } case MethodCallExpression { Method.Name: "Contains", Method.DeclaringType: Type dt, Object: MemberExpression member } methodCall when dt == typeof(string) && methodCall.Arguments is [ConstantExpression { Value: not null } argExpr]: { _ = MapPropertyToGoogleFilter(member.Member.Name) ?? throw new NotSupportedException( $"Property '{member.Member.Name}' is not supported for Google API negated Contains filters."); filters.Add(("excludeTerms", argExpr.Value)); break; } default: throw new NotSupportedException( $"Negated expression type '{expression.NodeType}' is not supported for Google API filters. " + $"Supported patterns: {string.Join(", ", s_supportedPatterns)}."); } } /// /// Maps GoogleWebPage property names to Google Custom Search API filter field names. /// /// The GoogleWebPage property name. /// The corresponding Google API filter name, or null if not mappable. private static string? MapPropertyToGoogleFilter(string propertyName) => propertyName.ToUpperInvariant() switch { // Map GoogleWebPage properties to Google API equivalents "LINK" => "siteSearch", // Maps to site search "DISPLAYLINK" => "siteSearch", // Maps to site search "TITLE" => "exactTerms", // Exact title match "SNIPPET" => "exactTerms", // Exact content match // Direct API parameters mapped from GoogleWebPage metadata properties "FILEFORMAT" => "fileType", // File type/extension filtering // Note: GoogleWebPage does expose a Mime property, but the Google Custom Search 'filter' // API parameter does not accept MIME types; it only accepts 0 or 1 for duplicate // content filtering. Therefore, no Mime-to-filter mapping is defined here. // Note: HL, GL, CR, LR properties don't exist on GoogleWebPage _ => null // Property not mappable to Google filters }; #endregion /// public void Dispose() { this._search.Dispose(); } private readonly ILogger _logger; private readonly CustomSearchAPIService _search; private readonly string? _searchEngineId; private readonly ITextSearchStringMapper _stringMapper; private readonly ITextSearchResultMapper _resultMapper; private readonly GoogleTextSearchOptions? _options; private static readonly ITextSearchStringMapper s_defaultStringMapper = new DefaultTextSearchStringMapper(); private static readonly ITextSearchResultMapper s_defaultResultMapper = new DefaultTextSearchResultMapper(); private const int MaxCount = 10; // See https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list private static readonly string[] s_queryParameters = ["cr", "dateRestrict", "exactTerms", "excludeTerms", "fileType", "filter", "gl", "hl", "linkSite", "lr", "orTerms", "rights", "siteSearch"]; // Performance optimization: Static error message arrays to avoid allocations in error paths private static readonly string[] s_supportedPatterns = [ "page.Property == \"value\" (exact match)", "page.Property != \"value\" (exclude)", "page.Property.Contains(\"text\") (partial match)", "!page.Property.Contains(\"text\") (exclude partial)", "page.Prop1 == \"val1\" && page.Prop2.Contains(\"val2\") (compound AND)" ]; private delegate void SetSearchProperty(CseResource.ListRequest search, string value); private static readonly Dictionary s_searchPropertySetters = new() { { "CR", (search, value) => search.Cr = value }, { "DATERESTRICT", (search, value) => search.DateRestrict = value }, { "EXACTTERMS", (search, value) => search.ExactTerms = value }, { "EXCLUDETERMS", (search, value) => search.ExcludeTerms = value }, { "FILETYPE", (search, value) => search.FileType = value }, { "FILTER", (search, value) => search.Filter = value }, { "GL", (search, value) => search.Gl = value }, { "HL", (search, value) => search.Hl = value }, { "LINKSITE", (search, value) => search.LinkSite = value }, { "LR", (search, value) => search.Lr = value }, { "ORTERMS", (search, value) => search.OrTerms = value }, { "RIGHTS", (search, value) => search.Rights = value }, { "SITESEARCH", (search, value) => { search.SiteSearch = value; search.SiteSearchFilter = CseResource.ListRequest.SiteSearchFilterEnum.I; } }, }; /// /// Execute a Google search /// /// The query string. /// Number of results to return. /// Number of results to skip. /// Pre-extracted filter key-value pairs. /// A cancellation token to cancel the request. /// /// private async Task ExecuteSearchAsync(string query, int top, int skip, List<(string FieldName, object Value)> filters, CancellationToken cancellationToken) { if (top <= 0 || top > MaxCount) { throw new ArgumentOutOfRangeException(nameof(top), top, $"{nameof(top)} value must be greater than 0 and less than or equals 10."); } if (skip < 0) { throw new ArgumentOutOfRangeException(nameof(skip), skip, $"{nameof(skip)} value must be greater than or equal to 0."); } var search = this._search.Cse.List(); search.Cx = this._searchEngineId; search.Q = query; search.Num = top; search.Start = skip; this.ApplyFilters(search, filters); return await search.ExecuteAsync(cancellationToken).ConfigureAwait(false); } /// /// Extracts filter key-value pairs from a legacy . /// This shim converts the obsolete FilterClause-based format to the internal (FieldName, Value) list. /// It will be removed when the legacy ITextSearch interface is retired. /// #pragma warning disable CS0618 // Obsolete TextSearchFilter, FilterClause private static List<(string FieldName, object Value)> ExtractFiltersFromLegacy(TextSearchFilter? filter) { var filters = new List<(string FieldName, object Value)>(); if (filter is not null) { foreach (var clause in filter.FilterClauses) { if (clause is EqualToFilterClause eq) { filters.Add((eq.FieldName, eq.Value)); } else { throw new NotSupportedException( $"Filter clause type '{clause.GetType().Name}' is not supported by Google Text Search. Only EqualToFilterClause is supported."); } } } return filters; } #pragma warning restore CS0618 /// /// Apply pre-extracted filter key-value pairs to the Google search request. /// Both LINQ and legacy paths converge here after producing the same (FieldName, Value) list. /// /// Google search metadata /// Pre-extracted filter key-value pairs. private void ApplyFilters(CseResource.ListRequest search, List<(string FieldName, object Value)> filters) { HashSet processedParams = new(StringComparer.OrdinalIgnoreCase); foreach (var (fieldName, value) in filters) { if (value is not string stringValue) { continue; } if (s_searchPropertySetters.TryGetValue(fieldName.ToUpperInvariant(), out var setter)) { setter.Invoke(search, stringValue); processedParams.Add(fieldName); } else { throw new ArgumentException($"Unknown equality filter clause field name '{fieldName}', must be one of {string.Join(",", s_queryParameters)}", nameof(filters)); } } // Apply default search parameters from constructor options (only for params not already set by filter) this.ApplyDefaultSearchParameters(search, processedParams); } /// /// Applies default Google search parameters from to the search request. /// Parameters already set by filter clauses are not overridden. /// private void ApplyDefaultSearchParameters(CseResource.ListRequest search, HashSet processedParams) { if (this._options is null) { return; } if (this._options.CountryRestrict is not null && !processedParams.Contains("cr")) { search.Cr = this._options.CountryRestrict; } if (this._options.DateRestrict is not null && !processedParams.Contains("dateRestrict")) { search.DateRestrict = this._options.DateRestrict; } if (this._options.GeoLocation is not null && !processedParams.Contains("gl")) { search.Gl = this._options.GeoLocation; } if (this._options.InterfaceLanguage is not null && !processedParams.Contains("hl")) { search.Hl = this._options.InterfaceLanguage; } if (this._options.LinkSite is not null && !processedParams.Contains("linkSite")) { search.LinkSite = this._options.LinkSite; } if (this._options.LanguageRestrict is not null && !processedParams.Contains("lr")) { search.Lr = this._options.LanguageRestrict; } if (this._options.Rights is not null && !processedParams.Contains("rights")) { search.Rights = this._options.Rights; } if (this._options.DuplicateContentFilter is not null && !processedParams.Contains("filter")) { search.Filter = this._options.DuplicateContentFilter; } } /// /// Return the search results as instances of . /// /// Google search response /// Cancellation token private async IAsyncEnumerable GetResultsAsTextSearchResultAsync(global::Google.Apis.CustomSearchAPI.v1.Data.Search searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) { if (searchResponse is null || searchResponse.Items is null) { yield break; } foreach (var item in searchResponse.Items) { yield return this._resultMapper.MapFromResultToTextSearchResult(item); await Task.Yield(); } } /// /// Return the search results as instances of . /// /// Google search response /// Cancellation token private async IAsyncEnumerable GetResultsAsStringAsync(global::Google.Apis.CustomSearchAPI.v1.Data.Search searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) { if (searchResponse is null || searchResponse.Items is null) { yield break; } foreach (var item in searchResponse.Items) { yield return this._stringMapper.MapFromResultToString(item); await Task.Yield(); } } /// /// Return the search results as instances of . /// /// Google search response /// Cancellation token private async IAsyncEnumerable GetResultsAsGoogleWebPageAsync(global::Google.Apis.CustomSearchAPI.v1.Data.Search searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) { if (searchResponse is null || searchResponse.Items is null) { yield break; } foreach (var item in searchResponse.Items) { yield return ConvertToGoogleWebPage(item); await Task.Yield(); } } /// /// Return the search results as instances of . /// /// Google search response /// Cancellation token private async IAsyncEnumerable GetResultsAsResultAsync(global::Google.Apis.CustomSearchAPI.v1.Data.Search searchResponse, [EnumeratorCancellation] CancellationToken cancellationToken) { if (searchResponse is null || searchResponse.Items is null) { yield break; } foreach (var item in searchResponse.Items) { yield return item; await Task.Yield(); } } /// /// Return the results metadata. /// /// Google search response private static Dictionary? GetResultsMetadata(global::Google.Apis.CustomSearchAPI.v1.Data.Search searchResponse) { return new Dictionary() { { "ETag", searchResponse.ETag }, }; } /// /// Converts a Google CustomSearchAPI Result to a GoogleWebPage instance. /// /// The Google search result to convert. /// A GoogleWebPage with mapped properties. private static GoogleWebPage ConvertToGoogleWebPage(global::Google.Apis.CustomSearchAPI.v1.Data.Result googleResult) { return new GoogleWebPage { Title = googleResult.Title, Link = googleResult.Link, Snippet = googleResult.Snippet, DisplayLink = googleResult.DisplayLink, FormattedUrl = googleResult.FormattedUrl, HtmlFormattedUrl = googleResult.HtmlFormattedUrl, HtmlSnippet = googleResult.HtmlSnippet, HtmlTitle = googleResult.HtmlTitle, Mime = googleResult.Mime, FileFormat = googleResult.FileFormat, Labels = googleResult.Labels?.Select(l => l.Name).ToArray() }; } /// /// Default implementation which maps from a to a /// private sealed class DefaultTextSearchStringMapper : ITextSearchStringMapper { /// public string MapFromResultToString(object result) { if (result is not global::Google.Apis.CustomSearchAPI.v1.Data.Result googleResult) { throw new ArgumentException("Result must be a Google Result", nameof(result)); } return googleResult.Snippet ?? string.Empty; } } /// /// Default implementation which maps from a to a /// private sealed class DefaultTextSearchResultMapper : ITextSearchResultMapper { /// public TextSearchResult MapFromResultToTextSearchResult(object result) { if (result is not global::Google.Apis.CustomSearchAPI.v1.Data.Result googleResult) { throw new ArgumentException("Result must be a Google Result", nameof(result)); } return new TextSearchResult(googleResult.Snippet) { Name = googleResult.Title, Link = googleResult.Link }; } } }