{ "$schema": "https://raw.githubusercontent.com/guitaripod/wwdc-sessions/master/schema/session.schema.json", "schemaVersion": 1, "id": "wwdc2026-297", "event": "wwdc2026", "eventName": "WWDC26", "year": 2026, "eventContentId": 297, "title": "Best practices for integrating visual intelligence in your app", "description": "Gain insight on how visual intelligence can transform content discovery in your app. Explore how to define entities, process images, and handle multiple result types effectively. Learn best practices for optimizing speed and relevance, and discover how intents enable direct actions like opening or playing content with a single tap.", "url": "https://developer.apple.com/videos/play/wwdc2026/297", "sosumiURL": "https://sosumi.ai/videos/play/wwdc2026/297", "type": "Video", "publishedAt": "2026-06-08T14:00:00-07:00", "platforms": [ "iOS", "iPadOS", "macOS" ], "keywords": [ "ai", "intelligence", "machine learning", "visual", "xcode" ], "topics": [ "App Services", "AI & Machine Learning" ], "primaryTopic": "AI & Machine Learning", "duration": 1065, "hasTranscript": true, "transcriptWordCount": 2316, "availableLanguages": [ "eng", "fra", "jpn", "kor", "por", "spa", "zho" ], "media": { "hls": "https://devstreaming-cdn.apple.com/videos/wwdc/2026/297/5/25343020-b502-4808-967a-6f6460789dc2/cmaf.m3u8", "downloadHLS": "https://devstreaming-cdn.apple.com/videos/wwdc/2026/297/5/25343020-b502-4808-967a-6f6460789dc2/cmaf-download.m3u8", "downloadSD": "https://devstreaming-cdn.apple.com/videos/wwdc/2026/297/5/25343020-b502-4808-967a-6f6460789dc2/downloads/wwdc2026-297_sd.mp4", "downloadHD": "https://devstreaming-cdn.apple.com/videos/wwdc/2026/297/5/25343020-b502-4808-967a-6f6460789dc2/downloads/wwdc2026-297_hd.mp4", "duration": 1065 }, "codeSnippets": [ { "title": "Define the content you want to return as an App Entity", "language": "swift", "startTimeSeconds": 201, "code": "// Define the content you want to return as an App Entity\n import AppIntents\n\n struct AlbumEntity: AppEntity {\n var id: String\n @Property var name: String\n @Property var artistName: String\n var coverArtData: Data\n\n var displayRepresentation: DisplayRepresentation {\n DisplayRepresentation( \n title: \"\\(name)\",\n subtitle: \"\\(artistName)\",\n image: .init(data: coverArtData)\n ) \n } \n\n static let defaultQuery = AlbumEntityQuery()\n\n static var typeDisplayRepresentation: TypeDisplayRepresentation { \"Album\" }\n } \n\n struct AlbumEntityQuery: EntityQuery {\n @Dependency var catalog: AlbumCatalog\n func entities(for identifiers: [String]) async throws -> [AlbumEntity] {\n catalog.albums(for: identifiers)\n }\n }" }, { "title": "Adopt IntentValueQuery to return results", "language": "swift", "startTimeSeconds": 339, "code": "// Adopt IntentValueQuery to return visual search results\n import AppIntents\n import VisualIntelligence\n\n struct SearchHandler: IntentValueQuery {\n @Dependency var catalog: AlbumCatalog\n @Dependency var concertFinder: ConcertFinder\n\n func values(for input: SemanticContentDescriptor) async throws -> [VisualSearchResult] {\n guard let pixelBuffer = input.pixelBuffer else {\n return []\n } \n\n let albums = try await catalog.search(matching: pixelBuffer)\n\n return albums.map { VisualSearchResult.album($0) }\n }\n }" }, { "title": "Build a catalog of albums with precomputed feature prints", "language": "swift", "startTimeSeconds": 384, "code": "// Build a catalog of albums with precomputed feature prints\n import Vision\n\n @Observable\n class AlbumCatalog {\n static let shared = AlbumCatalog()\n\n struct CatalogEntry: Sendable {\n let album: AlbumEntity\n let featurePrint: FeaturePrintObservation\n } \n\n private(set) var entries: [CatalogEntry] = []\n\n private func generateFeaturePrint(\n for image: CGImage\n ) async throws -> FeaturePrintObservation {\n let request = GenerateImageFeaturePrintRequest()\n let result = try await request.perform(on: image)\n return result\n }\n }" }, { "title": "Search the catalog for albums matching the captured image", "language": "swift", "startTimeSeconds": 405, "code": "// Search the catalog for albums matching the captured image\n func search(matching pixelBuffer: CVReadOnlyPixelBuffer, limit: Int = 10, maxDistance: Double = 1.0) async throws ->\n [AlbumEntity] {\n var cgImage: CGImage?\n _ = pixelBuffer.withUnsafeBuffer { VTCreateCGImageFromCVPixelBuffer($0, options: nil, imageOut: &cgImage) }\n guard let cgImage else { return [] }\n\n let queryPrint = try await generateFeaturePrint(for: cgImage)\n\n return try entries.compactMap { entry -> (album: AlbumEntity, distance: Double)? in\n let distance = try queryPrint.distance(to: entry.featurePrint)\n guard distance <= maxDistance else { return nil }\n return (entry.album, distance)\n } \n .sorted { $0.distance < $1.distance }\n .prefix(limit)\n .map { $0.album }\n }" }, { "title": "Create an open intent to land users on the right screen", "language": "swift", "startTimeSeconds": 507, "code": "// Create an open intent to land users on the right screen\n import AppIntents\n\n struct OpenAlbumIntent: OpenIntent {\n static let title: LocalizedStringResource = \"Open Album\"\n\n @Parameter(title: \"Album\")\n var target: AlbumEntity\n\n @Dependency var appState: AppState\n\n func perform() async throws -> some IntentResult {\n await appState.openAlbum(id: target.id)\n return .result()\n }\n }" }, { "title": "Use UnionValue to return multiple visual search result types", "language": "swift", "startTimeSeconds": 725, "code": "// Use UnionValue to return multiple visual search result types\n @UnionValue\n enum VisualSearchResult {\n case album(AlbumEntity)\n case concert(ConcertEntity)\n } \n\n struct OpenConcertIntent: OpenIntent {\n static let title: LocalizedStringResource = \"Open Concert\"\n\n @Parameter(title: \"Concert\")\n var target: ConcertEntity\n\n @Dependency var appState: AppState\n\n func perform() async throws -> some IntentResult {\n await appState.openConcert(id: target.id)\n return .result()\n }\n }" }, { "title": "Expand the IntentValueQuery to return the UnionValue", "language": "swift", "startTimeSeconds": 738, "code": "// Expand the IntentValueQuery to return the UnionValue\n struct SearchHandler: IntentValueQuery {\n @Dependency var catalog: AlbumCatalog\n @Dependency var concertFinder: ConcertFinder\n\n func values(for input: SemanticContentDescriptor) async throws -> [VisualSearchResult] {\n guard let pixelBuffer = input.pixelBuffer else {\n return []\n } \n\n let albums = try await catalog.search(matching: pixelBuffer)\n\n let artists = albums.map { $0.artistName }\n\n let concerts = await concertFinder.findNearby(byArtists: artists)\n\n return albums.map { VisualSearchResult.album($0) }\n + concerts.map { VisualSearchResult.concert($0) }\n }\n }" }, { "title": "Provide a link to in-app search", "language": "swift", "startTimeSeconds": 793, "code": "// Provide a link to in-app search\n @AppIntent(schema: .visualIntelligence.semanticContentSearch)\n struct SemanticContentSearchIntent: AppIntent {\n static let title: LocalizedStringResource = \"Search in app\"\n static let openAppWhenRun: Bool = true\n\n var semanticContent: SemanticContentDescriptor\n @Dependency var catalog: AlbumCatalog\n @Dependency var concertFinder: ConcertFinder\n @Dependency var appState: AppState\n\n func perform() async throws -> some IntentResult {\n guard let pixelBuffer = semanticContent.pixelBuffer else { return .result() }\n let albums = try await catalog.search(matching: pixelBuffer)\n let artists = albums.map { $0.artistName }\n let concerts = await concertFinder.findNearby(byArtists: artists)\n await appState.openSearch(albums: albums, concerts: concerts)\n return .result()\n } \n }" }, { "title": "Request calendar access and fetch upcoming concerts", "language": "swift", "startTimeSeconds": 924, "code": "// Request calendar access and fetch upcoming concerts\n import EventKit\n\n @Observable\n class UpcomingConcertManager {\n private let eventStore = EKEventStore()\n var upcomingConcerts: [EKEvent] = []\n var authorizationStatus: EKAuthorizationStatus = .notDetermined\n\n func requestAccessAndFetch() async throws {\n let granted = try await eventStore.requestFullAccessToEvents()\n guard granted else {\n authorizationStatus = .denied\n return\n } \n authorizationStatus = .fullAccess\n await fetchUpcomingConcerts()\n\n // ...\n }\n }" }, { "title": "Filter for upcoming events that match known artists in our catalog", "language": "swift", "startTimeSeconds": 942, "code": "// Filter for upcoming events that match known artists in our catalog\n class UpcomingConcertManager {\n func fetchUpcomingConcerts() async {\n let predicate = eventStore.predicateForEvents(\n withStart: .now,\n end: .now.addingTimeInterval(90 * 24 * 60 * 60),\n calendars: nil\n ) \n\n let events = eventStore.events(matching: predicate)\n\n upcomingConcerts = events.filter { event in\n AlbumCatalog.shared.entries.contains { entry in\n event.title?.localizedCaseInsensitiveContains(entry.album.artistName) == true\n }\n }\n }\n }" }, { "title": "Observe newly created events", "language": "swift", "startTimeSeconds": 944, "code": "// Observe newly created events\n @Observable\n class UpcomingConcertManager {\n // ...\n\n func requestAccessAndFetch() async throws {\n // ...\n\n for await _ in NotificationCenter.default\n .notifications(\n named: .EKEventStoreChanged\n ) {\n await fetchUpcomingConcerts()\n }\n }\n }" } ], "resources": [ { "title": "Integrating your app with visual intelligence", "url": "https://developer.apple.com/documentation/VisualIntelligence/integrating-your-app-with-visual-intelligence", "type": "documentation", "description": "Enable people to find app content that matches their surroundings or objects onscreen with visual intelligence.", "sosumiURL": "https://sosumi.ai/documentation/VisualIntelligence/integrating-your-app-with-visual-intelligence", "doccJSON": "https://developer.apple.com/tutorials/data/documentation/VisualIntelligence/integrating-your-app-with-visual-intelligence.json" }, { "title": "Visual Intelligence", "url": "https://developer.apple.com/documentation/VisualIntelligence", "type": "documentation", "description": "Include your app’s content in search results that visual intelligence provides.", "sosumiURL": "https://sosumi.ai/documentation/VisualIntelligence", "doccJSON": "https://developer.apple.com/tutorials/data/documentation/VisualIntelligence.json" } ], "path": "sessions/wwdc2026/297-best-practices-for-integrating-visual-intelligence-in-your-a/", "files": { "metadata": "sessions/wwdc2026/297-best-practices-for-integrating-visual-intelligence-in-your-a/metadata.json", "readme": "sessions/wwdc2026/297-best-practices-for-integrating-visual-intelligence-in-your-a/README.md", "transcript": "sessions/wwdc2026/297-best-practices-for-integrating-visual-intelligence-in-your-a/transcript.md", "transcriptJSON": "sessions/wwdc2026/297-best-practices-for-integrating-visual-intelligence-in-your-a/transcript.json" } }