// Copyright IBM Corp. 2025, 2026 // SPDX-License-Identifier: BUSL-1.1 scenario "plugin" { description = <<-EOF The plugin scenario deploys a Vault cluster with external integration services and runs comprehensive plugin blackbox tests. This scenario validates plugin functionality including: - LDAP secrets engine: Static/dynamic roles, password policies, rotation, rollback scenarios - Future plugins: Database, SSH, PKI, and other secrets engines The scenario creates dedicated external services (LDAP, databases, etc.) using containers and configures them with test data required for comprehensive plugin testing. # How to run this scenario For general instructions on running a scenario, refer to the Enos docs: https://eng-handbook.hashicorp.services/internal-tools/enos/running-a-scenario/ For troubleshooting tips and common errors, see https://eng-handbook.hashicorp.services/internal-tools/enos/troubleshooting/. Variables required for all scenario variants: - aws_ssh_private_key_path (more info about AWS SSH keypairs: https://eng-handbook.hashicorp.services/internal-tools/enos/getting-started/#set-your-aws-key-pair-name-and-private-key) - aws_ssh_keypair_name - vault_build_date* - vault_product_version - vault_revision* * If you don't already know what build date and revision you should be using, see https://eng-handbook.hashicorp.services/internal-tools/enos/troubleshooting/#execution-error-expected-vs-got-for-vault-versioneditionrevisionbuild-date. Variables required for some scenario variants: - artifactory_token (if using `artifact_source:artifactory` in your filter) - aws_region (if different from the default value in enos-variables.hcl) - vault_artifact_path (the path to where you have a Vault artifact already downloaded, if using `artifact_source:crt` in your filter) - vault_license_path (if using an ENT edition of Vault) EOF matrix { arch = global.archs artifact_source = global.artifact_sources artifact_type = global.artifact_types backend = global.backends config_mode = global.config_modes consul_edition = global.consul_editions consul_version = global.consul_versions distro = global.distros_aws edition = global.editions ip_version = global.ip_versions seal = global.seals // Our local builder always creates bundles exclude { artifact_source = ["local"] artifact_type = ["package"] } // PKCS#11 can only be used on ent.hsm and ent.hsm.fips1403. exclude { seal = ["pkcs11"] edition = [for e in matrix.edition : e if !strcontains(e, "hsm")] } // softhsm packages not available for sles (at the time of development) exclude { seal = ["pkcs11"] distro = ["sles"] } // Testing in IPV6 mode is currently implemented for integrated Raft storage only exclude { ip_version = ["6"] backend = ["consul"] } // plugin scenario cannot be run in CE exclude { edition = ["ce"] } } terraform_cli = terraform_cli.default terraform = terraform.default providers = [ provider.aws.default, provider.enos.ec2_user, provider.enos.ubuntu, provider.time.default, ] locals { artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null enos_provider = { amzn = provider.enos.ec2_user rhel = provider.enos.ec2_user sles = provider.enos.ec2_user ubuntu = provider.enos.ubuntu } manage_service = matrix.artifact_type == "bundle" } step "build_vault" { description = global.description.build_vault module = "build_${matrix.artifact_source}" variables { build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition] artifact_path = local.artifact_path goarch = matrix.arch goos = "linux" artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null arch = matrix.artifact_source == "artifactory" ? matrix.arch : null product_version = var.vault_product_version artifact_type = matrix.artifact_type distro = matrix.artifact_source == "artifactory" ? matrix.distro : null edition = matrix.artifact_source == "artifactory" ? matrix.edition : null revision = var.vault_revision } } step "ec2_info" { description = global.description.ec2_info module = module.ec2_info } step "create_vpc" { description = global.description.create_vpc module = module.create_vpc variables { common_tags = global.tags ip_version = matrix.ip_version } } step "read_backend_license" { description = global.description.read_backend_license module = module.read_license skip_step = matrix.backend == "raft" || matrix.consul_edition == "ce" variables { file_name = global.backend_license_path } } step "read_vault_license" { description = global.description.read_vault_license skip_step = matrix.edition == "ce" module = module.read_license variables { file_name = global.vault_license_path } } step "create_seal_key" { description = global.description.create_seal_key module = "seal_${matrix.seal}" depends_on = [step.create_vpc] providers = { enos = provider.enos.ubuntu } variables { cluster_id = step.create_vpc.id common_tags = global.tags } } step "create_plugin_integration_target" { description = "Create dedicated EC2 instance for external plugin services (LDAP, databases, etc.)" module = module.target_ec2_instances depends_on = [step.create_vpc] providers = { enos = provider.enos.ubuntu } variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["26.04"] cluster_tag_key = "plugin-integration" common_tags = global.tags instance_count = 1 root_volume_size = 64 vpc_id = step.create_vpc.id } } step "create_vault_cluster_targets" { description = global.description.create_vault_cluster_targets module = module.target_ec2_instances depends_on = [step.create_vpc] providers = { enos = local.enos_provider[matrix.distro] } variables { ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]] cluster_tag_key = global.vault_tag_key common_tags = global.tags seal_key_names = step.create_seal_key.resource_names vpc_id = step.create_vpc.id } } step "create_vault_cluster_backend_targets" { description = global.description.create_vault_cluster_targets module = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim depends_on = [step.create_vpc] providers = { enos = provider.enos.ubuntu } variables { ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"][global.distro_version["ubuntu"]] cluster_tag_key = global.backend_tag_key common_tags = global.tags seal_key_names = step.create_seal_key.resource_names vpc_id = step.create_vpc.id } } step "set_up_plugin_services" { description = "Set up external plugin services (LDAP server, databases, etc.) for plugin testing" module = module.set_up_external_integration_target depends_on = [ step.create_vault_cluster_targets ] providers = { enos = provider.enos.ubuntu } variables { hosts = step.create_plugin_integration_target.hosts ip_version = matrix.ip_version packages = concat(global.packages, global.distro_packages["ubuntu"]["26.04"], ["podman", "podman-docker"]) ports = global.integration_host_ports database_configs = global.database_configs } } step "create_backend_cluster" { description = global.description.create_backend_cluster module = "backend_${matrix.backend}" depends_on = [ step.create_vault_cluster_backend_targets ] providers = { enos = provider.enos.ubuntu } verifies = [ // verified in modules quality.consul_autojoin_aws, quality.consul_config_file, quality.consul_ha_leader_election, quality.consul_service_start_server, // verified in enos_consul_start resource quality.consul_api_agent_host_read, quality.consul_api_health_node_read, quality.consul_api_operator_raft_config_read, quality.consul_cli_validate, quality.consul_health_state_passing_read_nodes_minimum, quality.consul_operator_raft_configuration_read_voters_minimum, quality.consul_service_systemd_notified, quality.consul_service_systemd_unit, ] variables { cluster_name = step.create_vault_cluster_backend_targets.cluster_name cluster_tag_key = global.backend_tag_key hosts = step.create_vault_cluster_backend_targets.hosts license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null release = { edition = matrix.consul_edition version = matrix.consul_version } } } step "create_vault_cluster" { description = global.description.create_vault_cluster module = module.vault_cluster depends_on = [ step.create_backend_cluster, step.build_vault, step.create_vault_cluster_targets, step.set_up_plugin_services ] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ // verified in modules quality.consul_service_start_client, quality.vault_artifact_bundle, quality.vault_artifact_deb, quality.vault_artifact_rpm, quality.vault_audit_log, quality.vault_audit_socket, quality.vault_audit_syslog, quality.vault_autojoin_aws, quality.vault_config_env_variables, quality.vault_config_file, quality.vault_config_log_level, quality.vault_init, quality.vault_license_required_ent, quality.vault_listener_ipv4, quality.vault_listener_ipv6, quality.vault_service_start, quality.vault_storage_backend_consul, quality.vault_storage_backend_raft, // verified in enos_vault_start resource quality.vault_api_sys_config_read, quality.vault_api_sys_ha_status_read, quality.vault_api_sys_health_read, quality.vault_api_sys_host_info_read, quality.vault_api_sys_replication_status_read, quality.vault_api_sys_seal_status_api_read_matches_sys_health, quality.vault_api_sys_storage_raft_autopilot_configuration_read, quality.vault_api_sys_storage_raft_autopilot_state_read, quality.vault_api_sys_storage_raft_configuration_read, quality.vault_cli_status_exit_code, quality.vault_service_systemd_notified, quality.vault_service_systemd_unit, ] variables { artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name backend_cluster_tag_key = global.backend_tag_key cluster_name = step.create_vault_cluster_targets.cluster_name config_mode = matrix.config_mode consul_license = (matrix.backend == "consul" && matrix.consul_edition == "ent") ? step.read_backend_license.license : null consul_release = matrix.backend == "consul" ? { edition = matrix.consul_edition version = matrix.consul_version } : null enable_audit_devices = var.vault_enable_audit_devices hosts = step.create_vault_cluster_targets.hosts install_dir = global.vault_install_dir[matrix.artifact_type] ip_version = matrix.ip_version license = matrix.edition != "ce" ? step.read_vault_license.license : null local_artifact_path = local.artifact_path manage_service = local.manage_service packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = matrix.backend } } step "get_local_metadata" { description = global.description.get_local_metadata skip_step = matrix.artifact_source != "local" module = module.get_local_metadata } // Wait for our cluster to elect a leader step "wait_for_leader" { description = global.description.wait_for_cluster_to_have_leader module = module.vault_wait_for_leader depends_on = [step.create_vault_cluster] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ quality.vault_api_sys_leader_read, quality.vault_unseal_ha_leader_election, ] variables { timeout = 120 // seconds ip_version = matrix.ip_version hosts = step.create_vault_cluster_targets.hosts vault_addr = step.create_vault_cluster.api_addr_localhost vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_root_token = step.create_vault_cluster.root_token } } step "get_vault_cluster_ips" { description = global.description.get_vault_cluster_ip_addresses module = module.vault_get_cluster_ips depends_on = [step.wait_for_leader] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ quality.vault_api_sys_ha_status_read, quality.vault_api_sys_leader_read, quality.vault_cli_operator_members, ] variables { hosts = step.create_vault_cluster_targets.hosts ip_version = matrix.ip_version vault_addr = step.create_vault_cluster.api_addr_localhost vault_install_dir = global.vault_install_dir[matrix.artifact_type] vault_root_token = step.create_vault_cluster.root_token } } step "verify_vault_unsealed" { description = global.description.verify_vault_unsealed module = module.vault_wait_for_cluster_unsealed depends_on = [step.wait_for_leader] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ quality.vault_seal_awskms, quality.vault_seal_pkcs11, quality.vault_seal_shamir, ] variables { hosts = step.create_vault_cluster_targets.hosts vault_addr = step.create_vault_cluster.api_addr_localhost vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } step "run_verify_blackbox_tests" { description = global.description.run_verify_blackbox_tests module = module.vault_run_blackbox_test depends_on = [step.verify_vault_unsealed, step.get_vault_cluster_ips] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ quality.vault_api_sys_version_history_keys, quality.vault_api_sys_version_history_key_info, quality.vault_version_build_date, quality.vault_version_edition, quality.vault_version_release, ] variables { leader_host = step.get_vault_cluster_ips.leader_host leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token test_package = "./vault/external_tests/blackbox/isolated/verify" test_names = ["TestVaultServerVersion"] vault_edition = matrix.edition vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date vault_install_dir = global.vault_install_dir[matrix.artifact_type] } } locals { // Determine if filter contains test names (starts with "Test") or package names is_test_name_filter = length(var.blackbox_test_filter) > 0 && length([for t in var.blackbox_test_filter : t if can(regex("^Test", t))]) > 0 // For plugins, if package filter is provided, convert to plugin paths, otherwise run default plugin and system tests plugin_test_packages = length(var.blackbox_test_filter) > 0 && !local.is_test_name_filter ? [ for pkg in var.blackbox_test_filter : "./vault/external_tests/blackbox/isolated/plugins/${pkg}/..." ] : ["./vault/external_tests/blackbox/isolated/plugins/..."] } // Run comprehensive plugin blackbox tests step "run_plugin_blackbox_tests" { description = local.is_test_name_filter ? "Run specific plugin tests: ${join(", ", var.blackbox_test_filter)}" : "Run plugin blackbox tests from: ${join(", ", length(var.blackbox_test_filter) > 0 && !local.is_test_name_filter ? var.blackbox_test_filter : ["plugins"])}" module = module.vault_run_blackbox_test depends_on = [step.get_vault_cluster_ips, step.set_up_plugin_services, step.run_verify_blackbox_tests] providers = { enos = local.enos_provider[matrix.distro] } verifies = [ quality.vault_secrets_ldap_write_config, quality.vault_secrets_kmip_write_config, quality.vault_secrets_kv_read, quality.vault_secrets_kv_write, quality.vault_mount_auth, quality.vault_mount_kv, ] variables { leader_host = step.get_vault_cluster_ips.leader_host leader_public_ip = step.get_vault_cluster_ips.leader_public_ip vault_root_token = step.create_vault_cluster.root_token test_names = local.is_test_name_filter ? var.blackbox_test_filter : null test_package = local.is_test_name_filter ? "./vault/external_tests/blackbox" : join(" ", local.plugin_test_packages) integration_host_state = step.set_up_plugin_services.state vault_edition = matrix.edition vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision } } output "audit_device_file_path" { description = "The file path for the file audit device, if enabled" value = step.create_vault_cluster.audit_device_file_path } output "plugin_services_state" { description = "The external plugin services configuration and state" value = step.set_up_plugin_services.state } output "cluster_name" { description = "The Vault cluster name" value = step.create_vault_cluster.cluster_name } output "hosts" { description = "The Vault cluster target hosts" value = step.create_vault_cluster.hosts } output "private_ips" { description = "The Vault cluster private IPs" value = step.create_vault_cluster.private_ips } output "public_ips" { description = "The Vault cluster public IPs" value = step.create_vault_cluster.public_ips } output "root_token" { description = "The Vault cluster root token" value = step.create_vault_cluster.root_token } output "unseal_keys_b64" { description = "The Vault cluster unseal keys" value = step.create_vault_cluster.unseal_keys_b64 } output "unseal_keys_hex" { description = "The Vault cluster unseal keys hex" value = step.create_vault_cluster.unseal_keys_hex } output "plugin_test_results" { description = "Results from plugin blackbox tests" value = step.run_plugin_blackbox_tests.test_results_summary } }