{ "prefect": { "prefect-agent": { "type": "prefect-agent", "documentation_url": "https://docs.prefect.io/latest/concepts/work-pools/#agent-overview", "display_name": "Prefect Agent", "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/c771bb53894c877e169c8db158c5598558b8f175-24x24.svg", "install_command": "pip install prefect", "default_base_job_configuration": {}, "description": "Execute flow runs on heterogeneous infrastructure using infrastructure blocks." }, "process": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "stream_output": "{{ stream_output }}", "working_dir": "{{ working_dir }}" }, "variables": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "stream_output": { "default": true, "description": "If enabled, workers will stream output from flow run processes to local standard output.", "title": "Stream Output", "type": "boolean" }, "working_dir": { "anyOf": [ { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "If provided, workers will open flow run processes within the specified path as the working directory. Otherwise, a temporary directory will be created.", "title": "Working Directory" } }, "type": "object" } }, "description": "Execute flow runs as subprocesses on a worker. Works well for local execution when first getting started.", "display_name": "Process", "documentation_url": "https://docs.prefect.io/latest/get-started/quickstart", "install_command": "pip install prefect", "is_beta": false, "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/356e6766a91baf20e1d08bbe16e8b5aaef4d8643-48x48.png", "type": "process" } }, "prefect-aws": { "ecs": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "aws_credentials": "{{ aws_credentials }}", "task_definition": { "containerDefinitions": [ { "image": "{{ image }}", "name": "{{ container_name }}" } ], "cpu": "{{ cpu }}", "executionRoleArn": "{{ execution_role_arn }}", "family": "{{ family }}", "memory": "{{ memory }}" }, "task_run_request": { "capacityProviderStrategy": "{{ capacity_provider_strategy }}", "cluster": "{{ cluster }}", "launchType": "{{ launch_type }}", "overrides": { "containerOverrides": [ { "command": "{{ command }}", "cpu": "{{ cpu }}", "environment": "{{ env }}", "memory": "{{ memory }}", "name": "{{ container_name }}" } ], "cpu": "{{ cpu }}", "memory": "{{ memory }}", "taskRoleArn": "{{ task_role_arn }}" }, "tags": "{{ labels }}", "taskDefinition": "{{ task_definition_arn }}" }, "configure_cloudwatch_logs": "{{ configure_cloudwatch_logs }}", "cloudwatch_logs_options": "{{ cloudwatch_logs_options }}", "cloudwatch_logs_prefix": "{{ cloudwatch_logs_prefix }}", "network_configuration": "{{ network_configuration }}", "stream_output": false, "task_start_timeout_seconds": 0, "task_watch_poll_interval": 0, "auto_deregister_task_definition": "{{ auto_deregister_task_definition }}", "vpc_id": "{{ vpc_id }}", "container_name": "{{ container_name }}", "cluster": "{{ cluster }}", "match_latest_revision_in_family": "{{ match_latest_revision_in_family }}", "prefect_api_key_secret_arn": "{{ prefect_api_key_secret_arn }}", "prefect_api_auth_string_secret_arn": "{{ prefect_api_auth_string_secret_arn }}", "execution_role_arn": "{{ execution_role_arn }}" }, "variables": { "description": "Variables for templating an ECS job.", "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to provide to the task run. These variables are set on the Prefect container at task runtime. These will not be set on the task definition.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "task_definition_arn": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "An identifier for an existing task definition to use. If set, options that require changes to the task definition will be ignored. All contents of the task definition in the job configuration will be ignored.", "title": "Task Definition ARN" }, "aws_credentials": { "$ref": "#/definitions/AwsCredentials", "description": "The AWS credentials to use to connect to ECS. If not provided, credentials will be inferred from the local environment following AWS's boto client's rules.", "title": "AWS Credentials" }, "cluster": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The ECS cluster to run the task in. An ARN or name may be provided. If not provided, the default cluster will be used.", "title": "Cluster" }, "family": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "A family for the task definition. If not provided, it will be inferred from the task definition. If the task definition does not have a family, the name will be generated. When flow and deployment metadata is available, the generated name will include their names. Values for this field will be slugified to match AWS character requirements.", "title": "Family" }, "launch_type": { "default": "FARGATE", "description": "The type of ECS task run infrastructure that should be used. Note that 'FARGATE_SPOT' is not a formal ECS launch type, but we will configure the proper capacity provider strategy if set here.", "enum": [ "FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT" ], "title": "Launch Type", "type": "string" }, "capacity_provider_strategy": { "description": "The capacity provider strategy to use when running the task. If a capacity provider strategy is specified, the selected launch type will be ignored.", "items": { "$ref": "#/definitions/CapacityProvider" }, "title": "Capacity Provider Strategy", "type": "array" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The image to use for the Prefect container in the task. If this value is not null, it will override the value in the task definition. This value defaults to a Prefect base image matching your local versions.", "title": "Image" }, "cpu": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The amount of CPU to provide to the ECS task. Valid amounts are specified in the AWS documentation. If not provided, a default value of 1024 will be used unless present on the task definition.", "title": "CPU" }, "memory": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The amount of memory to provide to the ECS task. Valid amounts are specified in the AWS documentation. If not provided, a default value of 2048 will be used unless present on the task definition.", "title": "Memory" }, "container_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The name of the container flow run orchestration will occur in. If not specified, a default value of prefect will be used and if that is not found in the task definition the first container will be used.", "title": "Container Name" }, "prefect_api_key_secret_arn": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "An ARN of an AWS secret containing a Prefect API key. This key will be used to authenticate ECS tasks with Prefect Cloud. If not provided, the PREFECT_API_KEY environment variable will be used if the worker has one.", "title": "Prefect API Key Secret ARN" }, "prefect_api_auth_string_secret_arn": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "An ARN of an AWS secret containing a Prefect API auth string. This string will be used to authenticate ECS tasks with Prefect Cloud. If not provided, the PREFECT_API_AUTH_STRING environment variable will be used if the worker has one.", "title": "Prefect API Auth String Secret ARN" }, "task_role_arn": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "A role to attach to the task run. This controls the permissions of the task while it is running.", "title": "Task Role ARN" }, "execution_role_arn": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "An execution role to use for the task. This controls the permissions of the task when it is launching. If this value is not null, it will override the value in the task definition. An execution role must be provided to capture logs from the container.", "title": "Execution Role ARN" }, "vpc_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The AWS VPC to link the task run to. This is only applicable when using the 'awsvpc' network mode for your task. FARGATE tasks require this network mode, but for EC2 tasks the default network mode is 'bridge'. If using the 'awsvpc' network mode and this field is null, your default VPC will be used. If no default VPC can be found, the task run will fail.", "title": "VPC ID" }, "configure_cloudwatch_logs": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "If enabled, the Prefect container will be configured to send its output to the AWS CloudWatch logs service. This functionality requires an execution role with logs:CreateLogStream, logs:CreateLogGroup, and logs:PutLogEvents permissions. The default for this field is `False`.", "title": "Configure Cloudwatch Logs" }, "cloudwatch_logs_options": { "additionalProperties": { "type": "string" }, "description": "When `configure_cloudwatch_logs` is enabled, this setting may be used to pass additional options to the CloudWatch logs configuration or override the default options. See the [AWS documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_awslogs.html#create_awslogs_logdriver_options) for available options. ", "title": "Cloudwatch Logs Options", "type": "object" }, "cloudwatch_logs_prefix": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "When `configure_cloudwatch_logs` is enabled, this setting may be used to set a prefix for the log group. If not provided, the default prefix will be `prefect-logs__`. If `awslogs-stream-prefix` is present in `Cloudwatch logs options` this setting will be ignored.", "title": "Cloudwatch Logs Prefix" }, "network_configuration": { "additionalProperties": true, "description": "When `network_configuration` is supplied it will override ECS Worker'sawsvpcConfiguration that defined in the ECS task executing your workload. See the [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-service-awsvpcconfiguration.html) for available options.", "title": "Network Configuration", "type": "object" }, "auto_deregister_task_definition": { "default": false, "description": "If enabled, any task definitions that are created by this block will be deregistered. Existing task definitions linked by ARN will never be deregistered. Deregistering a task definition does not remove it from your AWS account, instead it will be marked as INACTIVE.", "title": "Auto Deregister Task Definition", "type": "boolean" }, "match_latest_revision_in_family": { "default": false, "description": "If enabled, the most recent active revision in the task definition family will be compared against the desired ECS task configuration. If they are equal, the existing task definition will be used instead of registering a new one. If no family is specified the default family \"prefect\" will be used.", "title": "Match Latest Revision In Family", "type": "boolean" } }, "type": "object", "definitions": { "AwsClientParameters": { "description": "Model used to manage extra parameters that you can pass when you initialize\nthe Client. If you want to find more information, see\n[boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html)\nfor more info about the possible client configurations.\n\nAttributes:\n api_version: The API version to use. By default, botocore will\n use the latest API version when creating a client. You only need\n to specify this parameter if you want to use a previous API version\n of the client.\n use_ssl: Whether or not to use SSL. By default, SSL is used.\n Note that not all services support non-ssl connections.\n verify: Whether or not to verify SSL certificates. By default\n SSL certificates are verified. If False, SSL will still be used\n (unless use_ssl is False), but SSL certificates\n will not be verified. Passing a file path to this is deprecated.\n verify_cert_path: A filename of the CA cert bundle to\n use. You can specify this argument if you want to use a\n different CA cert bundle than the one used by botocore.\n endpoint_url: The complete URL to use for the constructed\n client. Normally, botocore will automatically construct the\n appropriate URL to use when communicating with a service. You\n can specify a complete URL (including the \"http/https\" scheme)\n to override this behavior. If this value is provided,\n then ``use_ssl`` is ignored.\n config: Advanced configuration for Botocore clients. See\n [botocore docs](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html)\n for more details.", "properties": { "api_version": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The API version to use.", "title": "API Version" }, "use_ssl": { "default": true, "description": "Whether or not to use SSL.", "title": "Use SSL", "type": "boolean" }, "verify": { "anyOf": [ { "type": "boolean" }, { "format": "file-path", "type": "string" }, { "type": "null" } ], "default": null, "description": "Whether or not to verify SSL certificates.", "title": "Verify" }, "verify_cert_path": { "anyOf": [ { "format": "file-path", "type": "string" }, { "type": "null" } ], "default": null, "description": "Path to the CA cert bundle to use.", "title": "Certificate Authority Bundle File Path" }, "endpoint_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The complete URL to use for the constructed client.", "title": "Endpoint URL" }, "config": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "Advanced configuration for Botocore clients.", "title": "Botocore Config" } }, "title": "AwsClientParameters", "type": "object" }, "AwsCredentials": { "block_schema_references": {}, "block_type_slug": "aws-credentials", "description": "Block used to manage authentication with AWS. AWS authentication is\nhandled via the `boto3` module. Refer to the\n[boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)\nfor more info about the possible credential configurations.", "properties": { "aws_access_key_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "A specific AWS access key ID.", "title": "AWS Access Key ID" }, "aws_secret_access_key": { "anyOf": [ { "format": "password", "type": "string", "writeOnly": true }, { "type": "null" } ], "default": null, "description": "A specific AWS secret access key.", "title": "AWS Access Key Secret" }, "aws_session_token": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The session key for your AWS account. This is only needed when you are using temporary credentials.", "title": "AWS Session Token" }, "profile_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The profile to use when creating your session.", "title": "Profile Name" }, "region_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The AWS Region where you want to create new connections.", "title": "Region Name" }, "aws_client_parameters": { "$ref": "#/definitions/AwsClientParameters", "description": "Extra parameters to initialize the Client.", "title": "AWS Client Parameters" } }, "secret_fields": [ "aws_secret_access_key" ], "title": "AwsCredentials", "type": "object" }, "CapacityProvider": { "description": "The capacity provider strategy to use when running the task.", "properties": { "capacityProvider": { "title": "Capacityprovider", "type": "string" }, "weight": { "title": "Weight", "type": "integer" }, "base": { "title": "Base", "type": "integer" } }, "required": [ "capacityProvider", "weight", "base" ], "title": "CapacityProvider", "type": "object" } } } }, "description": "Execute flow runs within containers on AWS ECS. Works with EC2 and Fargate clusters. Requires an AWS account.", "display_name": "AWS Elastic Container Service", "documentation_url": "https://docs.prefect.io/integrations/prefect-aws/", "install_command": "pip install prefect-aws", "is_beta": false, "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/d74b16fe84ce626345adf235a47008fea2869a60-225x225.png", "type": "ecs" } }, "prefect-azure": { "azure-container-instance": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "image": "{{ image }}", "resource_group_name": "{{ resource_group_name }}", "subscription_id": "{{ subscription_id }}", "identities": "{{ identities }}", "entrypoint": "{{ entrypoint }}", "image_registry": "{{ image_registry }}", "cpu": "{{ cpu }}", "gpu_count": "{{ gpu_count }}", "gpu_sku": "{{ gpu_sku }}", "memory": "{{ memory }}", "subnet_ids": "{{ subnet_ids }}", "dns_servers": "{{ dns_servers }}", "stream_output": "{{ stream_output }}", "aci_credentials": "{{ aci_credentials }}", "task_start_timeout_seconds": "{{ task_start_timeout_seconds }}", "task_watch_poll_interval": "{{ task_watch_poll_interval }}", "arm_template": { "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "container_group_name": { "defaultValue": "[uniqueString(resourceGroup().id)]", "metadata": { "description": "The name of the container group to create." }, "type": "string" }, "container_name": { "defaultValue": "[uniqueString(resourceGroup().id)]", "metadata": { "description": "The name of the container to create." }, "type": "string" }, "location": { "defaultValue": "[resourceGroup().location]", "metadata": { "description": "Location for all resources." }, "type": "string" } }, "resources": [ { "apiVersion": "2022-09-01", "location": "[parameters('location')]", "name": "[parameters('container_group_name')]", "properties": { "containers": [ { "name": "[parameters('container_name')]", "properties": { "image": "{{ image }}", "command": "{{ command }}", "resources": { "requests": { "cpu": "{{ cpu }}", "memoryInGB": "{{ memory }}" } }, "environmentVariables": [] } } ], "osType": "Linux", "restartPolicy": "Never" }, "type": "Microsoft.ContainerInstance/containerGroups" } ] }, "keep_container_group": "{{ keep_container_group }}" }, "variables": { "description": "Variables for an Azure Container Instance flow run.", "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "image": { "description": "The image to use for the Prefect container in the task. This value defaults to a Prefect base image matching your local versions.", "title": "Image", "type": "string" }, "resource_group_name": { "description": "The name of the Azure Resource Group in which to run Prefect ACI tasks.", "title": "Azure Resource Group Name", "type": "string" }, "subscription_id": { "description": "The ID of the Azure subscription to create containers under.", "format": "password", "title": "Azure Subscription ID", "type": "string", "writeOnly": true }, "identities": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "A list of user-assigned identities to associate with the container group. The identities should be an ARM resource IDs in the form: '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}'.", "title": "Identities" }, "entrypoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "/opt/prefect/entrypoint.sh", "description": "The entrypoint of the container you wish you run. This value defaults to the entrypoint used by Prefect images and should only be changed when using a custom image that is not based on an official Prefect image. Any commands set on deployments will be passed to the entrypoint as parameters.", "title": "Entrypoint" }, "image_registry": { "anyOf": [ { "$ref": "#/definitions/ACRManagedIdentity" }, { "$ref": "#/definitions/DockerRegistryCredentials" }, { "type": "null" } ], "default": null, "description": "To use any private container registry with a username and password, choose DockerRegistry. To use a private Azure Container Registry with a managed identity, choose ACRManagedIdentity.", "title": "Image Registry (Optional)" }, "cpu": { "default": 1.0, "description": "The number of virtual CPUs to assign to the task container. If not provided, a default value of 1.0 will be used.", "title": "CPU", "type": "number" }, "gpu_count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of GPUs to assign to the task container. If not provided, no GPU will be used.", "title": "GPU Count" }, "gpu_sku": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The Azure GPU SKU to use. See the ACI documentation for a list of GPU SKUs available in each Azure region.", "title": "GPU SKU" }, "memory": { "default": 1.0, "description": "The amount of memory in gigabytes to provide to the ACI task. Valid amounts are specified in the Azure documentation. If not provided, a default value of 1.0 will be used unless present on the task definition.", "title": "Memory", "type": "number" }, "subnet_ids": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "A list of subnet IDs to associate with the container group. ", "title": "Subnet IDs" }, "dns_servers": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "A list of DNS servers to associate with the container group.", "title": "DNS Servers" }, "aci_credentials": { "$ref": "#/definitions/AzureContainerInstanceCredentials", "description": "The credentials to use to authenticate with Azure." }, "stream_output": { "default": false, "description": "If `True`, logs will be streamed from the Prefect container to the local console.", "title": "Stream Output", "type": "boolean" }, "task_start_timeout_seconds": { "default": 240, "description": "The amount of time to watch for the start of the ACI container. before marking it as failed.", "title": "Task Start Timeout Seconds", "type": "integer" }, "task_watch_poll_interval": { "default": 5.0, "description": "The number of seconds to wait between Azure API calls while monitoring the state of an Azure Container Instances task.", "title": "Task Watch Poll Interval", "type": "number" }, "keep_container_group": { "default": false, "description": "Keep the completed container group on Azure.", "title": "Keep Container Group After Completion", "type": "boolean" } }, "required": [ "resource_group_name", "subscription_id" ], "type": "object", "definitions": { "ACRManagedIdentity": { "description": "Use a Managed Identity to access Azure Container registry. Requires the\nuser-assigned managed identity be available to the ACI container group.", "properties": { "registry_url": { "description": "The URL to the registry, such as myregistry.azurecr.io. Generally, 'http' or 'https' can be omitted.", "title": "Registry URL", "type": "string" }, "identity": { "description": "The user-assigned Azure managed identity for the private registry.", "title": "Identity", "type": "string" } }, "required": [ "registry_url", "identity" ], "title": "ACRManagedIdentity", "type": "object" }, "AzureContainerInstanceCredentials": { "block_schema_references": {}, "block_type_slug": "azure-container-instance-credentials", "description": "Block used to manage Azure Container Instances authentication. Stores Azure Service\nPrincipal authentication data.", "properties": { "client_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The service principal client ID. If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential.", "title": "Client ID" }, "tenant_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The service principal tenant ID.If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential.", "title": "Tenant ID" }, "client_secret": { "anyOf": [ { "format": "password", "type": "string", "writeOnly": true }, { "type": "null" } ], "default": null, "description": "The service principal client secret.If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential.", "title": "Client Secret" }, "credential_kwargs": { "additionalProperties": true, "description": "Additional keyword arguments to pass to `ClientSecretCredential` or `DefaultAzureCredential`.", "title": "Additional Credential Keyword Arguments", "type": "object" } }, "secret_fields": [ "client_secret" ], "title": "AzureContainerInstanceCredentials", "type": "object" }, "DockerRegistryCredentials": { "block_schema_references": {}, "block_type_slug": "docker-registry-credentials", "description": "Store credentials for interacting with a Docker Registry.", "properties": { "username": { "description": "The username to log into the registry with.", "title": "Username", "type": "string" }, "password": { "description": "The password to log into the registry with.", "format": "password", "title": "Password", "type": "string", "writeOnly": true }, "registry_url": { "description": "The URL to the registry. Generally, \"http\" or \"https\" can be omitted.", "examples": [ "index.docker.io" ], "title": "Registry Url", "type": "string" }, "reauth": { "default": true, "description": "Whether or not to reauthenticate on each interaction.", "title": "Reauth", "type": "boolean" } }, "required": [ "username", "password", "registry_url" ], "secret_fields": [ "password" ], "title": "DockerRegistryCredentials", "type": "object" } } } }, "description": "Execute flow runs within containers on Azure's Container Instances service. Requires an Azure account.", "display_name": "Azure Container Instances", "documentation_url": "https://docs.prefect.io/integrations/prefect-azure", "install_command": "pip install prefect-azure", "is_beta": false, "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/54e3fa7e00197a4fbd1d82ed62494cb58d08c96a-250x250.png", "type": "azure-container-instance" } }, "prefect-docker": { "docker": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "image": "{{ image }}", "registry_credentials": "{{ registry_credentials }}", "image_pull_policy": "{{ image_pull_policy }}", "networks": "{{ networks }}", "network_mode": "{{ network_mode }}", "auto_remove": "{{ auto_remove }}", "volumes": "{{ volumes }}", "stream_output": "{{ stream_output }}", "mem_limit": "{{ mem_limit }}", "memswap_limit": "{{ memswap_limit }}", "privileged": "{{ privileged }}", "container_create_kwargs": "{{ container_create_kwargs }}" }, "variables": { "definitions": { "DockerRegistryCredentials": { "additionalProperties": true, "block_schema_references": {}, "block_type_slug": "docker-registry-credentials", "description": "Store credentials for interacting with a Docker Registry.", "properties": { "username": { "description": "The username to log into the registry with.", "title": "Username", "type": "string" }, "password": { "description": "The password to log into the registry with.", "format": "password", "title": "Password", "type": "string", "writeOnly": true }, "registry_url": { "description": "The URL to the registry. Generally, \"http\" or \"https\" can be omitted.", "examples": [ "index.docker.io" ], "title": "Registry Url", "type": "string" }, "reauth": { "default": true, "description": "Whether or not to reauthenticate on each interaction.", "title": "Reauth", "type": "boolean" } }, "required": [ "username", "password", "registry_url" ], "secret_fields": [ "password" ], "title": "DockerRegistryCredentials", "type": "object" } }, "description": "Configuration class used by the Docker worker.\n\nAn instance of this class is passed to the Docker worker's `run` method\nfor each flow run. It contains all the information necessary to execute the\nflow run as a Docker container.\n\nAttributes:\n name: The name to give to created Docker containers.\n command: The command executed in created Docker containers to kick off\n flow run execution.\n env: The environment variables to set in created Docker containers.\n labels: The labels to set on created Docker containers.\n image: The image reference of a container image to use for created jobs.\n If not set, the latest Prefect image will be used.\n image_pull_policy: The image pull policy to use when pulling images.\n networks: Docker networks that created containers should be connected to.\n network_mode: The network mode for the created containers (e.g. host, bridge).\n If 'networks' is set, this cannot be set.\n auto_remove: If set, containers will be deleted on completion.\n volumes: Docker volumes that should be mounted in created containers.\n stream_output: If set, the output from created containers will be streamed\n to local standard output.\n mem_limit: Memory limit of created containers. Accepts a value\n with a unit identifier (e.g. 100000b, 1000k, 128m, 1g.) If a value is\n given without a unit, bytes are assumed.\n memswap_limit: Total memory (memory + swap), -1 to disable swap. Should only be\n set if `mem_limit` is also set. If `mem_limit` is set, this defaults to\n allowing the container to use as much swap as memory. For example, if\n `mem_limit` is 300m and `memswap_limit` is not set, containers can use\n 600m in total of memory and swap.\n privileged: Give extended privileges to created containers.\n container_create_kwargs: Extra args for docker py when creating container.", "properties": { "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by the worker using this job configuration.", "title": "Labels", "type": "object" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by the worker using this job configuration.", "title": "Name" }, "image": { "description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.", "examples": [ "docker.io/prefecthq/prefect:3-latest" ], "title": "Image", "type": "string" }, "registry_credentials": { "anyOf": [ { "$ref": "#/definitions/DockerRegistryCredentials" }, { "type": "null" } ], "default": null, "description": "Credentials for logging into a Docker registry to pull images from." }, "image_pull_policy": { "anyOf": [ { "enum": [ "IfNotPresent", "Always", "Never" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "The image pull policy to use when pulling images.", "title": "Image Pull Policy" }, "networks": { "description": "Docker networks that created containers should be connected to.", "items": { "type": "string" }, "title": "Networks", "type": "array" }, "network_mode": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The network mode for the created containers (e.g. host, bridge). If 'networks' is set, this cannot be set.", "title": "Network Mode" }, "auto_remove": { "default": false, "description": "If set, containers will be deleted on completion.", "title": "Auto Remove", "type": "boolean" }, "volumes": { "description": "A list of volume to mount into created containers.", "examples": [ "/my/local/path:/path/in/container" ], "items": { "type": "string" }, "title": "Volumes", "type": "array" }, "stream_output": { "default": true, "description": "If set, the output from created containers will be streamed to local standard output.", "title": "Stream Output", "type": "boolean" }, "mem_limit": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Memory limit of created containers. Accepts a value with a unit identifier (e.g. 100000b, 1000k, 128m, 1g.) If a value is given without a unit, bytes are assumed.", "title": "Memory Limit" }, "memswap_limit": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Total memory (memory + swap), -1 to disable swap. Should only be set if `mem_limit` is also set. If `mem_limit` is set, this defaults toallowing the container to use as much swap as memory. For example, if `mem_limit` is 300m and `memswap_limit` is not set, containers can use 600m in total of memory and swap.", "title": "Memory Swap Limit" }, "privileged": { "default": false, "description": "Give extended privileges to created container.", "title": "Privileged", "type": "boolean" }, "container_create_kwargs": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "Configuration for containers created by workers. See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/containers.html) for accepted values.", "title": "Container Configuration" } }, "type": "object" } }, "description": "Execute flow runs within Docker containers. Works well for managing flow execution environments via Docker images. Requires access to a running Docker daemon.", "display_name": "Docker", "documentation_url": "https://docs.prefect.io/integrations/prefect-docker", "install_command": "pip install prefect-docker", "is_beta": false, "logo_url": "https://images.ctfassets.net/gm98wzqotmnx/2IfXXfMq66mrzJBDFFCHTp/6d8f320d9e4fc4393f045673d61ab612/Moby-logo.png?h=250", "type": "docker" } }, "prefect-gcp": { "cloud-run": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "region": "{{ region }}", "credentials": "{{ credentials }}", "prefect_api_key_secret": "{{ prefect_api_key_secret }}", "prefect_api_auth_string_secret": "{{ prefect_api_auth_string_secret }}", "job_body": { "apiVersion": "run.googleapis.com/v1", "kind": "Job", "metadata": { "annotations": { "run.googleapis.com/launch-stage": "BETA" }, "name": "{{ name }}" }, "spec": { "template": { "metadata": { "annotations": { "run.googleapis.com/vpc-access-connector": "{{ vpc_connector_name }}" } }, "spec": { "template": { "spec": { "containers": [ { "command": "{{ command }}", "image": "{{ image }}", "resources": { "limits": { "cpu": "{{ cpu }}", "memory": "{{ memory }}" }, "requests": { "cpu": "{{ cpu }}", "memory": "{{ memory }}" } } } ], "serviceAccountName": "{{ service_account_name }}", "timeoutSeconds": "{{ timeout }}" } } } } } }, "timeout": "{{ timeout }}", "keep_job": "{{ keep_job }}" }, "variables": { "description": "Default variables for the Cloud Run worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.", "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "region": { "default": "us-central1", "description": "The region where the Cloud Run Job resides.", "examples": [ "us-central1" ], "title": "Region", "type": "string" }, "credentials": { "anyOf": [ { "$ref": "#/definitions/GcpCredentials" }, { "type": "null" } ], "description": "The GCP Credentials used to initiate the Cloud Run Job. If not provided credentials will be inferred from the local environment.", "title": "GCP Credentials" }, "prefect_api_key_secret": { "anyOf": [ { "$ref": "#/definitions/SecretKeySelector" }, { "type": "null" } ], "default": null, "description": "A GCP secret containing a Prefect API Key. This key will be used to authenticate Cloud Run tasks with Prefect Cloud. If not provided, the PREFECT_API_KEY environment variable will be used if the worker has one.", "title": "Prefect API Key Secret" }, "prefect_api_auth_string_secret": { "anyOf": [ { "$ref": "#/definitions/SecretKeySelector" }, { "type": "null" } ], "default": null, "description": "A GCP secret containing a Prefect API authorization string. This string will be used to authenticate Cloud Run tasks with Prefect Cloud. If not provided, the PREFECT_API_AUTH_STRING environment variable will be used if the worker has one.", "title": "Prefect API Auth String Secret" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The image to use for a new Cloud Run Job. If not set, the latest Prefect image will be used. See https://cloud.google.com/run/docs/deploying#images.", "examples": [ "docker.io/prefecthq/prefect:3-latest" ], "title": "Image Name" }, "cpu": { "anyOf": [ { "pattern": "^(\\d*000)m$", "type": "string" }, { "type": "null" } ], "default": null, "description": "The amount of compute allocated to the Cloud Run Job. (1000m = 1 CPU). See https://cloud.google.com/run/docs/configuring/cpu#setting-jobs.", "examples": [ "1000m" ], "title": "CPU" }, "memory": { "anyOf": [ { "pattern": "^\\d+(?:G|Gi|M|Mi)$", "type": "string" }, { "type": "null" } ], "default": null, "description": "The amount of memory allocated to the Cloud Run Job. Must be specified in units of 'G', 'Gi', 'M', or 'Mi'. See https://cloud.google.com/run/docs/configuring/memory-limits#setting.", "examples": [ "512Mi" ], "title": "Memory" }, "vpc_connector_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The name of the VPC connector to use for the Cloud Run Job.", "title": "VPC Connector Name" }, "service_account_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The name of the service account to use for the task execution of Cloud Run Job. By default Cloud Run jobs run as the default Compute Engine Service Account. ", "examples": [ "service-account@example.iam.gserviceaccount.com" ], "title": "Service Account Name" }, "keep_job": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": false, "description": "Keep the completed Cloud Run Job after it has run.", "title": "Keep Job After Completion" }, "timeout": { "anyOf": [ { "exclusiveMinimum": 0, "maximum": 3600, "type": "integer" }, { "type": "null" } ], "default": 600, "description": "Max allowed duration the Job may be active before Cloud Run will actively try to mark it failed and kill associated containers (maximum of 3600 seconds, 1 hour).", "title": "Job Timeout" } }, "type": "object", "definitions": { "GcpCredentials": { "block_schema_references": {}, "block_type_slug": "gcp-credentials", "description": "Block used to manage authentication with GCP. Google authentication is\nhandled via the `google.oauth2` module or through the CLI.\nSpecify either one of service `account_file` or `service_account_info`; if both\nare not specified, the client will try to detect the credentials following Google's\n[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials).\nSee Google's [Authentication documentation](https://cloud.google.com/docs/authentication#service-accounts)\nfor details on inference and recommended authentication patterns.", "properties": { "service_account_file": { "anyOf": [ { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "Path to the service account JSON keyfile.", "title": "Service Account File" }, "service_account_info": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "The contents of the keyfile as a dict.", "title": "Service Account Info" }, "project": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The GCP project to use for the client.", "title": "Project" } }, "secret_fields": [ "service_account_info.*" ], "title": "GcpCredentials", "type": "object" }, "SecretKeySelector": { "description": "SecretKeySelector is a data model for specifying a GCP secret to inject\ninto a Cloud Run V2 Job as an environment variable.\nFollows Cloud Run V2 rest API, docs:\nhttps://cloud.google.com/run/docs/reference/rest/v2/Container#SecretKeySelector", "properties": { "secret": { "title": "Secret", "type": "string" }, "version": { "title": "Version", "type": "string" } }, "required": [ "secret", "version" ], "title": "SecretKeySelector", "type": "object" } } } }, "description": "Execute flow runs within containers on Google Cloud Run. Requires a Google Cloud Platform account.", "display_name": "Google Cloud Run", "documentation_url": "https://docs.prefect.io/integrations/prefect-gcp", "install_command": "pip install prefect-gcp", "is_beta": false, "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png", "type": "cloud-run" }, "cloud-run-v2": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "credentials": "{{ credentials }}", "env_from_secrets": "{{ env_from_secrets }}", "prefect_api_key_secret": "{{ prefect_api_key_secret }}", "prefect_api_auth_string_secret": "{{ prefect_api_auth_string_secret }}", "cloudsql_instances": "{{ cloudsql_instances }}", "job_body": { "client": "prefect", "launchStage": "{{ launch_stage }}", "template": { "template": { "containers": [ { "args": "{{ args }}", "command": "{{ command }}", "env": [], "image": "{{ image }}", "resources": { "limits": { "cpu": "{{ cpu }}", "memory": "{{ memory }}" } } } ], "maxRetries": "{{ max_retries }}", "serviceAccount": "{{ service_account_name }}", "timeout": "{{ timeout }}", "vpcAccess": { "connector": "{{ vpc_connector_name }}" } } } }, "keep_job": "{{ keep_job }}", "region": "{{ region }}", "timeout": "{{ timeout }}" }, "variables": { "description": "Default variables for the v2 Cloud Run worker.\n\nThe schema for this class is used to populate the `variables` section of the\ndefault base job template.", "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "credentials": { "$ref": "#/definitions/GcpCredentials", "description": "The GCP Credentials used to connect to Cloud Run. If not provided credentials will be inferred from the local environment.", "title": "GCP Credentials" }, "region": { "default": "us-central1", "description": "The region in which to run the Cloud Run job", "title": "Region", "type": "string" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "prefecthq/prefect:3-latest", "description": "The image to use for the Cloud Run job. If not provided the default Prefect image will be used.", "title": "Image Name" }, "args": { "description": "The arguments to pass to the Cloud Run Job V2's entrypoint command.", "items": { "type": "string" }, "title": "Args", "type": "array" }, "env_from_secrets": { "additionalProperties": { "$ref": "#/definitions/SecretKeySelector" }, "description": "Environment variables to set from GCP secrets when starting a flow run.", "examples": [ { "ENV_VAR_NAME": { "secret": "SECRET_NAME", "version": "latest" } } ], "title": "Environment Variables from Secrets", "type": "object" }, "prefect_api_key_secret": { "anyOf": [ { "$ref": "#/definitions/SecretKeySelector" }, { "type": "null" } ], "default": null, "description": "The GCP secret to use for the Prefect API key. When provided, the secret will be used instead of the PREFECT_API_KEY environment variable.", "examples": [ { "secret": "prefect-api-key", "version": "latest" } ], "title": "Prefect API Key Secret" }, "prefect_api_auth_string_secret": { "anyOf": [ { "$ref": "#/definitions/SecretKeySelector" }, { "type": "null" } ], "default": null, "description": "The GCP secret to use for the Prefect API auth string. When provided, the secret will be used instead of the PREFECT_API_AUTH_STRING environment variable.", "examples": [ { "secret": "prefect-api-auth-string", "version": "latest" } ], "title": "Prefect API Auth String Secret" }, "cloudsql_instances": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "description": "List of Cloud SQL instance connection names to connect to. Format: {project}:{location}:{instance}", "examples": [ "project:region:instance-id" ], "title": "Cloud SQL Instances" }, "keep_job": { "default": false, "description": "Keep the completed Cloud run job on Google Cloud Platform.", "title": "Keep Job After Completion", "type": "boolean" }, "launch_stage": { "default": "BETA", "description": "The launch stage of the Cloud Run Job V2. See https://cloud.google.com/run/docs/about-features-categories for additional details.", "enum": [ "ALPHA", "BETA", "GA", "DEPRECATED", "EARLY_ACCESS", "PRELAUNCH", "UNIMPLEMENTED", "LAUNCH_TAG_UNSPECIFIED" ], "title": "Launch Stage", "type": "string" }, "max_retries": { "default": 0, "description": "The number of times to retry the Cloud Run job.", "title": "Max Retries", "type": "integer" }, "cpu": { "default": "1000m", "description": "The CPU to allocate to the Cloud Run job.", "title": "CPU", "type": "string" }, "memory": { "default": "512Mi", "description": "The memory to allocate to the Cloud Run job along with the units, whichcould be: G, Gi, M, Mi.", "examples": [ "512Mi" ], "pattern": "^\\d+(?:G|Gi|M|Mi)$", "title": "Memory", "type": "string" }, "timeout": { "default": 600, "description": "Max allowed time duration the Job may be active before Cloud Run will actively try to mark it failed and kill associated containers (maximum of 86400 seconds, 1 day).", "exclusiveMinimum": 0, "maximum": 86400, "title": "Job Timeout", "type": "integer" }, "vpc_connector_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The name of the VPC connector to use for the Cloud Run job.", "title": "VPC Connector Name" }, "service_account_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The name of the service account to use for the task execution of Cloud Run Job. By default Cloud Run jobs run as the default Compute Engine Service Account.", "examples": [ "service-account@example.iam.gserviceaccount.com" ], "title": "Service Account Name" } }, "type": "object", "definitions": { "GcpCredentials": { "block_schema_references": {}, "block_type_slug": "gcp-credentials", "description": "Block used to manage authentication with GCP. Google authentication is\nhandled via the `google.oauth2` module or through the CLI.\nSpecify either one of service `account_file` or `service_account_info`; if both\nare not specified, the client will try to detect the credentials following Google's\n[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials).\nSee Google's [Authentication documentation](https://cloud.google.com/docs/authentication#service-accounts)\nfor details on inference and recommended authentication patterns.", "properties": { "service_account_file": { "anyOf": [ { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "Path to the service account JSON keyfile.", "title": "Service Account File" }, "service_account_info": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "The contents of the keyfile as a dict.", "title": "Service Account Info" }, "project": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The GCP project to use for the client.", "title": "Project" } }, "secret_fields": [ "service_account_info.*" ], "title": "GcpCredentials", "type": "object" }, "SecretKeySelector": { "description": "SecretKeySelector is a data model for specifying a GCP secret to inject\ninto a Cloud Run V2 Job as an environment variable.\nFollows Cloud Run V2 rest API, docs:\nhttps://cloud.google.com/run/docs/reference/rest/v2/Container#SecretKeySelector", "properties": { "secret": { "title": "Secret", "type": "string" }, "version": { "title": "Version", "type": "string" } }, "required": [ "secret", "version" ], "title": "SecretKeySelector", "type": "object" } } } }, "description": "Execute flow runs within containers on Google Cloud Run (V2 API). Requires a Google Cloud Platform account.", "display_name": "Google Cloud Run V2", "documentation_url": "https://docs.prefect.io/integrations/prefect-gcp", "install_command": "pip install prefect-gcp", "is_beta": false, "logo_url": "https://images.ctfassets.net/gm98wzqotmnx/4SpnOBvMYkHp6z939MDKP6/549a91bc1ce9afd4fb12c68db7b68106/social-icon-google-cloud-1200-630.png?h=250", "type": "cloud-run-v2" }, "vertex-ai": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "region": "{{ region }}", "credentials": "{{ credentials }}", "job_spec": { "enable_dashboard_access": "{{ enable_dashboard_access }}", "enable_web_access": "{{ enable_web_access }}", "maximum_run_time_hours": "{{ maximum_run_time_hours }}", "network": "{{ network }}", "reserved_ip_ranges": "{{ reserved_ip_ranges }}", "scheduling": "{{ scheduling }}", "service_account_name": "{{ service_account_name }}", "worker_pool_specs": [ { "container_spec": { "args": [], "command": "{{ command }}", "image_uri": "{{ image }}" }, "disk_spec": { "boot_disk_size_gb": "{{ boot_disk_size_gb }}", "boot_disk_type": "{{ boot_disk_type }}" }, "machine_spec": { "accelerator_count": "{{ accelerator_count }}", "accelerator_type": "{{ accelerator_type }}", "machine_type": "{{ machine_type }}" }, "replica_count": 1 } ] }, "job_watch_poll_interval": "{{ job_watch_poll_interval }}" }, "variables": { "description": "Default variables for the Vertex AI worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.", "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "region": { "description": "The region where the Vertex AI Job resides.", "examples": [ "us-central1" ], "title": "Region", "type": "string" }, "image": { "description": "The URI of a container image in the Container or Artifact Registry, used to run your Vertex AI Job. Note that Vertex AI will need accessto the project and region where the container image is stored. See https://cloud.google.com/vertex-ai/docs/training/create-custom-container", "examples": [ "gcr.io/your-project/your-repo:latest" ], "title": "Image Name", "type": "string" }, "credentials": { "anyOf": [ { "$ref": "#/definitions/GcpCredentials" }, { "type": "null" } ], "description": "The GCP Credentials used to initiate the Vertex AI Job. If not provided credentials will be inferred from the local environment.", "title": "GCP Credentials" }, "machine_type": { "default": "n1-standard-4", "description": "The machine type to use for the run, which controls the available CPU and memory. See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec", "title": "Machine Type", "type": "string" }, "accelerator_type": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The type of accelerator to attach to the machine. See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec", "examples": [ "NVIDIA_TESLA_K80" ], "title": "Accelerator Type" }, "accelerator_count": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of accelerators to attach to the machine. See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec", "examples": [ 1 ], "title": "Accelerator Count" }, "boot_disk_type": { "default": "pd-ssd", "description": "The type of boot disk to attach to the machine.", "title": "Boot Disk Type", "type": "string" }, "boot_disk_size_gb": { "default": 100, "description": "The size of the boot disk to attach to the machine, in gigabytes.", "title": "Boot Disk Size (GB)", "type": "integer" }, "maximum_run_time_hours": { "default": 1, "description": "The maximum job running time, in hours", "title": "Maximum Run Time (Hours)", "type": "integer" }, "network": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The full name of the Compute Engine networkto which the Job should be peered. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. For example: projects/12345/global/networks/myVPC", "title": "Network" }, "reserved_ip_ranges": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "A list of names for the reserved ip ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided ip ranges. Otherwise, the job will be deployed to any ip ranges under the provided VPC network.", "title": "Reserved IP Ranges" }, "scheduling": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "A dictionary with scheduling options for a CustomJob, these are parameters related to queuing, and scheduling custom jobs. If unspecified default scheduling options are used. The 'maximum_run_time_hours' variable sets the job timeout field 'scheduling.timeout' for backward compatibility. See SDK: https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.Scheduling See REST: https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#Scheduling", "examples": [ { "scheduling": { "max_wait_duration": "1800s", "strategy": "FLEX_START" } } ], "title": "Scheduling Options" }, "service_account_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Specifies the service account to use as the run-as account in Vertex AI. The worker submitting jobs must have act-as permission on this run-as account. If unspecified, the AI Platform Custom Code Service Agent for the CustomJob's project is used. Takes precedence over the service account found in GCP credentials, and required if a service account cannot be detected in GCP credentials.", "title": "Service Account Name" }, "enable_web_access": { "default": false, "description": "Whether you want Vertex AI to enable `interactive shell access` See https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell for how to access your job via interactive console when running.", "examples": [ true ], "title": "Enable Web Access", "type": "boolean" }, "enable_dashboard_access": { "default": false, "description": "Whether to enable access to the customized dashboard in the training chief container. See https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell#get-uri for access instructions.", "examples": [ true ], "title": "Enable Dashboard Access", "type": "boolean" }, "job_watch_poll_interval": { "default": 5.0, "description": "The amount of time to wait between GCP API calls while monitoring the state of a Vertex AI Job.", "title": "Poll Interval (Seconds)", "type": "number" } }, "required": [ "region", "image" ], "type": "object", "definitions": { "GcpCredentials": { "block_schema_references": {}, "block_type_slug": "gcp-credentials", "description": "Block used to manage authentication with GCP. Google authentication is\nhandled via the `google.oauth2` module or through the CLI.\nSpecify either one of service `account_file` or `service_account_info`; if both\nare not specified, the client will try to detect the credentials following Google's\n[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials).\nSee Google's [Authentication documentation](https://cloud.google.com/docs/authentication#service-accounts)\nfor details on inference and recommended authentication patterns.", "properties": { "service_account_file": { "anyOf": [ { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "Path to the service account JSON keyfile.", "title": "Service Account File" }, "service_account_info": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "The contents of the keyfile as a dict.", "title": "Service Account Info" }, "project": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The GCP project to use for the client.", "title": "Project" } }, "secret_fields": [ "service_account_info.*" ], "title": "GcpCredentials", "type": "object" } } } }, "description": "Execute flow runs within containers on Google Vertex AI. Requires a Google Cloud Platform account.", "display_name": "Google Vertex AI", "documentation_url": "https://docs.prefect.io/integrations/prefect-gcp", "install_command": "pip install prefect-gcp", "is_beta": false, "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png", "type": "vertex-ai" } }, "prefect-kubernetes": { "kubernetes": { "default_base_job_configuration": { "job_configuration": { "command": "{{ command }}", "env": "{{ env }}", "labels": "{{ labels }}", "name": "{{ name }}", "namespace": "{{ namespace }}", "job_manifest": { "apiVersion": "batch/v1", "kind": "Job", "metadata": { "generateName": "{{ name }}-", "labels": "{{ labels }}", "namespace": "{{ namespace }}" }, "spec": { "backoffLimit": "{{ backoff_limit }}", "template": { "spec": { "completions": 1, "containers": [ { "args": "{{ command }}", "env": "{{ env }}", "image": "{{ image }}", "imagePullPolicy": "{{ image_pull_policy }}", "name": "prefect-job" } ], "parallelism": 1, "restartPolicy": "Never", "serviceAccountName": "{{ service_account_name }}" } }, "ttlSecondsAfterFinished": "{{ finished_job_ttl }}" } }, "cluster_config": "{{ cluster_config }}", "job_watch_timeout_seconds": "{{ job_watch_timeout_seconds }}", "pod_watch_timeout_seconds": "{{ pod_watch_timeout_seconds }}", "stream_output": "{{ stream_output }}" }, "variables": { "description": "Default variables for the Kubernetes worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.", "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Name given to infrastructure created by a worker.", "title": "Name" }, "env": { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "description": "Environment variables to set when starting a flow run.", "title": "Environment Variables", "type": "object" }, "labels": { "additionalProperties": { "type": "string" }, "description": "Labels applied to infrastructure created by a worker.", "title": "Labels", "type": "object" }, "command": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.", "title": "Command" }, "namespace": { "default": "default", "description": "The Kubernetes namespace to create jobs within.", "title": "Namespace", "type": "string" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.", "examples": [ "docker.io/prefecthq/prefect:3-latest" ], "title": "Image" }, "service_account_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The Kubernetes service account to use for job creation.", "title": "Service Account Name" }, "image_pull_policy": { "default": "IfNotPresent", "description": "The Kubernetes image pull policy to use for job containers.", "enum": [ "IfNotPresent", "Always", "Never" ], "title": "Image Pull Policy", "type": "string" }, "backoff_limit": { "default": 0, "description": "The number of times Kubernetes will retry a job after pod eviction. If set to 0, Prefect will reschedule the flow run when the pod is evicted unless PREFECT_FLOW_RUN_EXECUTE_SIGTERM_BEHAVIOR is set to value different from 'reschedule'.", "minimum": 0, "title": "Backoff Limit", "type": "integer" }, "finished_job_ttl": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of seconds to retain jobs after completion. If set, finished jobs will be cleaned up by Kubernetes after the given delay. If not set, jobs will be retained indefinitely.", "title": "Finished Job TTL" }, "job_watch_timeout_seconds": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Number of seconds to wait for each event emitted by a job before timing out. If not set, the worker will wait for each event indefinitely.", "title": "Job Watch Timeout Seconds" }, "pod_watch_timeout_seconds": { "default": 60, "description": "Number of seconds to watch for pod creation before timing out.", "title": "Pod Watch Timeout Seconds", "type": "integer" }, "stream_output": { "default": true, "description": "If set, output will be streamed from the job to local standard output.", "title": "Stream Output", "type": "boolean" }, "cluster_config": { "anyOf": [ { "$ref": "#/definitions/KubernetesClusterConfig" }, { "type": "null" } ], "default": null, "description": "The Kubernetes cluster config to use for job creation." } }, "type": "object", "definitions": { "KubernetesClusterConfig": { "block_schema_references": {}, "block_type_slug": "kubernetes-cluster-config", "description": "Stores configuration for interaction with Kubernetes clusters.\n\nSee `from_file` for creation.", "properties": { "config": { "additionalProperties": true, "description": "The entire contents of a kubectl config file.", "title": "Config", "type": "object" }, "context_name": { "description": "The name of the kubectl context to use.", "title": "Context Name", "type": "string" } }, "required": [ "config", "context_name" ], "secret_fields": [], "title": "KubernetesClusterConfig", "type": "object" } } } }, "description": "Execute flow runs within jobs scheduled on a Kubernetes cluster. Requires a Kubernetes cluster.", "display_name": "Kubernetes", "documentation_url": "https://docs.prefect.io/integrations/prefect-kubernetes", "install_command": "pip install prefect-kubernetes", "is_beta": false, "logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/2d0b896006ad463b49c28aaac14f31e00e32cfab-250x250.png", "type": "kubernetes" } } }