# This Source Code Form is subject to the terms of the Mozilla Public↩
# License, v. 2.0. If a copy of the MPL was not distributed with this↩
# file, You can obtain one at http://mozilla.org/MPL/2.0/.↩

# Adding a new metric? We have docs for that!↩
# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html↩

---
$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
$tags:
  - "Core :: Machine Learning"

firefox.ai.runtime:
  session_start:
    type: event
    description: >
      Marks the beginning of an ML inference session or workflow.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - thasan@mozilla.com
    expires: never
    extra_keys:
      flow_id: &flow_id
        type: string
        description: >
          Unique identifier for this ML workflow session. Used to correlate all
          events within this session across components.
      feature_id:
        type: string
        description: >
          The feature initiating this session
      interaction:
        type: string
        description: >
          The interaction type that initiated this session

  session_end:
    type: event
    description: >
      Marks the end of an ML inference session or workflow.
      This event is recorded when the session completes successfully or is terminated.
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - thasan@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      feature_id:
        type: string
        description: >
          The feature associated with this session.
      duration:
        type: quantity
        description: >
          Total session duration in milliseconds.
      status:
        type: string
        description: >
          Session completion status

  engine_creation_success_flow:
    type: event
    description: >
      Records a successful engine creation within a tracked flow
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - thasan@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      engineId:
        type: string
        description: >
          The Engine Id of the created inference engine
      duration:
        type: quantity
        description: >
          Engine creation time in milliseconds

  engine_creation_success:
    type: labeled_timing_distribution
    time_unit: millisecond
    description: >
      An inference engine was created
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D243790
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
    expires: never
    labels:
      - autofill-ml
      - pdfjs
      - ml-suggest-intent
      - ml-suggest-ner
      - about-inference
      - webextension
      - wllamapreview
      - default-engine
      - smart-tab-embedding-engine
      - smart-tab-topic-engine
      - smart-intent

  engine_creation_failure:
    type: event
    description: >
      An inference engine has failed to create
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - tshasan@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      modelId:
        type: string
        description: model id
      featureId:
        description: feature id
        type: string
      taskName:
        description: taskName id
        type: string
      engineId:
        description: engine id
        type: string
      error:
        description: error class
        type: string

  run_inference_failure:
    type: event
    description: >
      An inference run has failed
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - tshasan@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      modelId:
        type: string
        description: model id
      featureId:
        description: feature id
        type: string
      engineId:
        description: Engine id
        type: string

  run_inference_success_flow:
    type: event
    description: >
      Records a successful inference run within a tracked flow
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - thasan@mozilla.com
    expires: never
    extra_keys:
      flow_id: *flow_id
      tokenizing_time:
        type: quantity
        description: >
          Time taken for tokenization in milliseconds
      inference_time:
        type: quantity
        description: >
          Time taken for inference in milliseconds

  run_inference_success:
    type: labeled_timing_distribution
    time_unit: millisecond
    description: >
      An inference engine was created
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
      - https://phabricator.services.mozilla.com/D243790
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
    expires: never
    labels:
      - autofill-ml
      - pdfjs
      - ml-suggest-intent
      - ml-suggest-ner
      - about-inference
      - webextension
      - wllamapreview
      - default-engine
      - smart-tab-embedding-engine
      - smart-tab-topic-engine
      - smart-intent

  model_download:
    type: event
    description: >
      A model is downloaded
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1891685
    data_reviews:
      - https://phabricator.services.mozilla.com/D238583
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
    expires: never
    extra_keys:
      modelDownloadId:
        description: used to identify models files downloaded together in the pipeline
        type: string
      featureId:
        description: feature id
        type: string
      engineId:
        type: string
        description: model id
      modelId:
        type: string
        description: model id
      step:
        description: start / cancel / error / complete
        type: string
      duration:
        description: how long it took to reach the step from start
        type: quantity
      when:
        description: when the event was recorded
        type: quantity
      modelRevision:
        description: model revision
        type: string
      error:
        description: error class
        type: string

  model_deletion:
    type: event
    description: >
      A model is deleted
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1966301
    data_reviews:
      - https://phabricator.services.mozilla.com/D249218
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
    expires: never
    extra_keys:
      deletedBy:
        type: string
        description: the name of the feature that deleted the model
      modelId:
        type: string
        description: model id
      modelRevision:
        description: model revision
        type: string
      error:
        description: error class
        type: string

  engine_run:
    type: event
    description: >
      The performance and memory characteristics of a single run of the inference engine.
    extra_keys:
      flow_id: *flow_id
      cpu_milliseconds:
        description: The combined milliseconds of every cpu core that was running.
        type: quantity
      wall_milliseconds:
        description: The amount of wall time the run request took.
        type: quantity
      cores:
        description: The number of cores on the machine.
        type: quantity
      cpu_utilization:
        description: The percentage of the user's CPU used.
        type: quantity
      memory_bytes:
        description: The number of RSS bytes for the inference process.
        type: quantity
      feature_id:
        description: >
          A unique id to describe the feature. e.g. pdfjs-alt-text, suggest-NER,
          link-preview.
        type: string
      engine_id:
        type: string
        description: >
          Similar to the feature id, but is used to ensure only one engine is created
          under this name.
      model_id:
        type: string
        description: >
          The identifier for what model is being used by the engine,
          e.g. Xenova/all-MiniLM-L6-v2
      backend:
        type: string
        description: The backend that is being used, e.g. onnx, onnx-native, wllama
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1992779
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997016
    data_reviews:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1992779
      - https://phabricator.services.mozilla.com/D270909
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - gtatum@mozilla.com
      - thasan@mozilla.com
    expires: never

model_management:
  remove_initiated:
    type: event
    description: |
      Recorded when the user clicks on Remove from the three dots next to
      a model item or the remove button from the model details page
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - ngrato@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      source:
        description: The location from which the link was clicked e.g., details, list
        type: string
      size:
        description: The size of the model in bytes
        type: quantity
      last_used:
        description: hourse since the model was used
        type: quantity
      last_install:
        description: hourse since the model was installed
        type: quantity
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never

  remove_confirmation:
    type: event
    description: |
      Recorded when the user interacts with the
      panel to confirm removal of model
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - ngrato@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
      action:
        description: The action that was taken by the user, remove or cancel
        type: string
    expires: never

  list_item_manage:
    type: event
    description: |
      Recorded when the user clicks on Manage from the three
      dots next to a model item
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - ngrato@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never

  model_card_link:
    type: event
    description: |
      Recorded when the user clicks the model card link in the model details
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - ngrato@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never

  list_view:
    type: event
    description: |
      Recorded when the user views the model management section in Add-ons Manager
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - ngrato@mozilla.com
    extra_keys:
      models:
        description: Number of models in the list
        type: quantity
    expires: never

  details_view:
    type: event
    description: |
      Recorded when the user views the screen with model details in the model management section in Add-ons Manager
    bugs:
      - https://bugzilla.mozilla.org/show_bug.cgi?id=1947256
    data_reviews:
      - https://phabricator.services.mozilla.com/D249462
    data_sensitivity:
      - interaction
    notification_emails:
      - tziade@mozilla.com
      - ngrato@mozilla.com
    extra_keys:
      extension_ids:
        description: Ids of the extensions that use this model
        type: string
      feature_ids:
        description: Ids of the engines that are linked to a feature
        type: string
      model:
        description: The id of the model
        type: string
      version:
        description: version of the current model
        type: string
    expires: never