# This Source Code Form is subject to the terms of the Mozilla Public↩
# License, v. 2.0. If a copy of the MPL was not distributed with this↩
# file, You can obtain one at http://mozilla.org/MPL/2.0/.↩

# Adding a new metric? We have docs for that!↩
# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html↩

---
$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
$tags:
  - "Core :: Machine Learning"

firefox.ai.runtime:
  session_start:
    type: event
    description: >
      Marks the beginning of an ML inference session or workflow.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      flow_id: &flow_id
        type: string
        description: >
          Unique identifier for this ML workflow session. Used to correlate all
          events within this session across components.
      feature_id:
        type: string
        description: >
          The feature initiating this session
      interaction:
        type: string
        description: >
          The interaction type that initiated this session

  session_end:
    type: event
    description: >
      Marks the end of an ML inference session or workflow.
      This event is recorded when the session completes successfully or is terminated.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      feature_id:
        type: string
        description: >
          The feature associated with this session.
      duration:
        type: quantity
        description: >
          Total session duration in milliseconds.
      status:
        type: string
        description: >
          Session completion status

  engine_creation_success_flow:
    type: event
    description: >
      Records a successful engine creation within a tracked flow
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      engineId:
        type: string
        description: >
          The Engine Id of the created inference engine
      duration:
        type: quantity
        description: >
          Engine creation time in milliseconds

  engine_creation_success:
    type: labeled_timing_distribution
    time_unit: millisecond
    description: >
      An inference engine was created
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D243790
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    labels: &engine_labels
      - autofill-ml
      - pdfjs
      - ml-suggest-intent
      - ml-suggest-ner
      - about-inference
      - webextension
      - link-preview
      - default-engine
      - smart-tab-embedding-engine
      - smart-tab-topic-engine
      - smart-intent
      - smart-openai
      - title-generation-engine
      - smart-openai-memories-generation
      - smart-openai-memories-usage

  engine_creation_failure:
    type: event
    description: >
      An inference engine has failed to create
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      modelId:
        type: string
        description: model id
      featureId:
        description: feature id
        type: string
      taskName:
        description: taskName id
        type: string
      engineId:
        description: engine id
        type: string
      error:
        description: error class
        type: string

  run_inference_failure:
    type: event
    description: >
      An inference run has failed
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      modelId:
        type: string
        description: model id
      featureId:
        description: feature id
        type: string
      engineId:
        description: Engine id
        type: string
      error:
        description: error class or message
        type: string

  run_inference_success_flow:
    type: event
    description: >
      Records a successful inference run within a tracked flow
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      tokenizing_time:
        type: quantity
        description: >
          Time taken for tokenization (preprocessing) in milliseconds
      inference_time:
        type: quantity
        description: >
          Time taken for inference in milliseconds
      decoding_time:
        type: quantity
        description: >
          Time taken for decoding in milliseconds
      input_tokens:
        type: quantity
        description: >
          Number of tokens in the input prompt
      output_tokens:
        type: quantity
        description: >
          Number of tokens generated
      time_to_first_token:
        type: quantity
        description: >
          Time to first token in milliseconds (null if not applicable)
      tokens_per_second:
        type: quantity
        description: >
          Inference throughput in tokens per second (rounded integer)
      time_per_output_token:
        type: quantity
        description: >
          Average latency per output token in milliseconds (rounded integer)

  run_inference_success:
    type: labeled_timing_distribution
    time_unit: millisecond
    description: >
      An inference engine was created
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D243790
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    labels: *engine_labels

  model_download:
    type: event
    description: >
      A model is downloaded
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2015825
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      modelDownloadId:
        description: used to identify models files downloaded together in the pipeline
        type: string
      featureId:
        description: feature id
        type: string
      engineId:
        type: string
        description: model id
      modelId:
        type: string
        description: model id
      step:
        description: >
          One of start_download, start_file_download,
          end_file_download_success, end_file_download_failed,
          end_download_success, end_download_failed.
        type: string
      duration:
        description: >
          Milliseconds elapsed to reach the step from its start. For
          end_file_download_* this is the time to download a single file; for
          end_download_* this is the total time to download the whole model.
        type: quantity
      when:
        description: when the event was recorded
        type: quantity
      modelRevision:
        description: model revision
        type: string
      error:
        description: error class
        type: string

  model_deletion:
    type: event
    description: >
      A model is deleted
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1966301
    data_reviews:
      - https://phabricator.services.mozilla.com/D249218
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never
    extra_keys:
      deletedBy:
        type: string
        description: the name of the feature that deleted the model
      modelId:
        type: string
        description: model id
      modelRevision:
        description: model revision
        type: string
      error:
        description: error class
        type: string

  engine_run:
    type: event
    description: >
      The performance and memory characteristics of a single run of the inference engine.
    extra_keys:
      flow_id: *flow_id
      cpu_milliseconds:
        description: The combined milliseconds of every cpu core that was running.
        type: quantity
      wall_milliseconds:
        description: The amount of wall time the run request took.
        type: quantity
      cores:
        description: The number of cores on the machine.
        type: quantity
      cpu_utilization:
        description: The percentage of the user's CPU used.
        type: quantity
      memory_bytes:
        description: The number of RSS bytes for the inference process.
        type: quantity
      feature_id:
        description: >
          A unique id to describe the feature. e.g. pdfjs-alt-text, suggest-NER,
          link-preview.
        type: string
      engine_id:
        type: string
        description: >
          Similar to the feature id, but is used to ensure only one engine is created
          under this name.
      model_id:
        type: string
        description: >
          The identifier for what model is being used by the engine,
          e.g. Xenova/all-MiniLM-L6-v2
      backend:
        type: string
        description: The backend that is being used, e.g. onnx, onnx-native, llama.cpp
      backend_source_revision:
        type: string
        description: >
          Version identifier of the backend library. Populated per-backend when
          the running library exposes a version; null otherwise.
      token_count:
        description: The number of tokens generated for text generation tasks.
        type: quantity
      character_count:
        description: The number of characters generated for text generation tasks.
        type: quantity
      time_to_first_chunk:
        description: >
          Milliseconds from the run request being dispatched until the first
          generated (non-prompt) chunk is received by the consumer. Measured in
          the parent process end-to-end, so it includes IPC and streaming
          overhead, and is populated for all streaming backends.
        type: quantity
      average_chunk_time:
        description: >
          The mean milliseconds elapsed between consecutive generated
          (non-prompt) chunks for a streamed text generation run. Null when
          fewer than two generated chunks were produced.
        type: quantity
      system_memory_mb:
        description: The total physical system memory in MB.
        type: quantity
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1992779
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2029250
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2016214
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2047026
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1992779
      - https://phabricator.services.mozilla.com/D270909
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2016214
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    expires: never

  onnx_native_availability:
    type: labeled_counter
    description: >
      Whether the native ONNX runtime could be loaded in the inference process,
      as reported by InferenceSession.isAvailable().
      It is recorded at most once per profile, guarded by the
      browser.ml.onnxNativeAvailabilityReported pref.
    labels:
      - available
      - unavailable
      - probe_error
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2063024
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=2063024
    data_sensitivity:
      - technical
    notification_emails:
      - vpollet@mozilla.com
      - enordin@mozilla.com
    expires: 160

model_management:
  remove_initiated:
    type: event
    description: |
      Recorded when the user clicks on Remove from the three dots next to
      a model item or the remove button from the model details page
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      source:
        description: The location from which the link was clicked e.g., details, list
        type: string
      size:
        description: The size of the model in bytes
        type: quantity
      last_used:
        description: hourse since the model was used
        type: quantity
      last_install:
        description: hourse since the model was installed
        type: quantity
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never

  remove_confirmation:
    type: event
    description: |
      Recorded when the user interacts with the
      panel to confirm removal of model
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
      action:
        description: The action that was taken by the user, remove or cancel
        type: string
    expires: never

  list_item_manage:
    type: event
    description: |
      Recorded when the user clicks on Manage from the three
      dots next to a model item
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never

  model_card_link:
    type: event
    description: |
      Recorded when the user clicks the model card link in the model details
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never

  list_view:
    type: event
    description: |
      Recorded when the user views the model management section in Add-ons Manager
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    extra_keys:
      models:
        description: Number of models in the list
        type: quantity
    expires: never

  details_view:
    type: event
    description: |
      Recorded when the user views the screen with model details in the model management section in Add-ons Manager
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - firefox-ai-and-ml@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never